The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
fix(p2p): give negotiation a budget that covers the peer's gather, and see where it goes
0xken15aad11bf08544
diff --git a/cli/src/config/env.ts b/cli/src/config/env.tsindex 4bf04c908..bab9cec15 100644--- a/cli/src/config/env.ts+++ b/cli/src/config/env.ts@@ -344,6 +344,10 @@ const envSchema = z.object({// encrypted on the wire, so a packet capture cannot answer "what did the adapter actually send".// Never prints message text, transcripts, tokens or payload bodies.LOG_FRAMES: z.string().default('false').transform((v) => v === 'true'),+ // TEMPORARY / diagnostic. Forces iceTransportPolicy:'relay', so the ONLY candidate either peer can+ // offer is a Cloudflare TURN allocation — the way to exercise the relay path on a pair that would+ // otherwise always connect directly. Costs money per GB while it is on; never ship it enabled.+ TERMINAL_P2P_FORCE_RELAY: z.string().default('false').transform((v) => v === 'true'),// ── OpenRouter gateway agents (`ori claude`, `ori codex`, …) ───────────────────────────────────// An agent whose CLI is pointed at OpenRouter has no vendor credential to spend, so its recap anddiff --git a/cli/src/lib/remoteRelay.spec.ts b/cli/src/lib/remoteRelay.spec.tsindex 3c95a1e15..696578259 100644--- a/cli/src/lib/remoteRelay.spec.ts+++ b/cli/src/lib/remoteRelay.spec.ts@@ -45,6 +45,36 @@ describe('RemoteRelayPool reports terminal_link_mode to the local sink', () => {})})+ it.each([+ ['relay', 'turn'],+ ['direct', 'p2p'],+ [null, 'p2p'],+ ])('a p2p terminal_ready whose ICE pair is %s reports mode:%s', (transport, expected) => {+ // The data channel is up either way; what differs is which candidate pair ICE nominated. A relay+ // pair means every byte is going through Cloudflare TURN, which is the state that costs money —+ // and the one the badge could not tell apart before. A null pair keeps the pre-TURN optimism.+ const entry = fakeEntry({ p2pPendingOpens: new Set(['req-turn']), p2p: { transport } })+ pool.noteTerminalResponse(entry, { type: 'terminal_ready', payload: { requestId: 'req-turn', streamId: 'stream-turn' } }, 'p2p')++ expect(entry.p2pStreams.has('stream-turn')).toBe(true)+ expect(entry.sink.sendFrame).toHaveBeenCalledWith({+ type: 'terminal_link_mode',+ payload: { streamId: 'stream-turn', mode: expected },+ })+ })++ it('a stream with no data channel stays mode:relay however the ICE pair reads', () => {+ // Guards the additive rename: 'relay' must keep meaning "on the backend WebSocket" so an older+ // Desktop build can never read a TURN session as a WS one.+ const entry = fakeEntry({ p2pPendingOpens: new Set(['req-ws']), p2p: { transport: 'relay' } })+ pool.noteTerminalResponse(entry, { type: 'terminal_ready', payload: { requestId: 'req-ws', streamId: 'stream-ws' } }, 'relay')++ expect(entry.sink.sendFrame).toHaveBeenCalledWith({+ type: 'terminal_link_mode',+ payload: { streamId: 'stream-ws', mode: 'relay' },+ })+ })+it('a terminal_ready delivered over relay reports mode:relay', () => {const entry = fakeEntry({ p2pPendingOpens: new Set(['req-2']) })pool.noteTerminalResponse(entry, { type: 'terminal_ready', payload: { requestId: 'req-2', streamId: 'stream-2' } }, 'relay')diff --git a/cli/src/lib/remoteRelay.ts b/cli/src/lib/remoteRelay.tsindex 8fa29fa83..66a0647d4 100644--- a/cli/src/lib/remoteRelay.ts+++ b/cli/src/lib/remoteRelay.ts@@ -16,6 +16,7 @@ import { WebSocket, type RawData } from 'ws'import type { Frame, LocalClientSink } from '../backendSocket.js'import type { AuthSessionManager } from './authSession.js'import { b64d, type Identity } from './e2ee/core.js'+import { sid } from './log.js'import type { MachinePeerStore } from './e2ee/machinePeers.js'import { RelaySessionCrypto } from './e2ee/relayClient.js'import { encodeTerminalLocal, TerminalBinaryKind, type TerminalBinaryClear } from './terminalBinary.js'@@ -232,7 +233,17 @@ export class RemoteRelayPool {// in-flight dedupe: begin() joins THIS race instead of starting a second one, and its// cost hides behind the welcome round trip. Nothing here is load-bearing for// correctness; drop it and the only change is a slightly later first terminal.- if (entry.p2pPolicy) warmStunUrls(entry.p2pPolicy.stunUrls)+ // Unconditional, one line per machine select. Whether the backend actually handed us a+ // TURN credential is invisible everywhere else — not in LOG_FRAMES (which only covers+ // backendSocket, never this relay client), not in the badge, not in p2p_result — and its+ // absence is indistinguishable from "TURN is configured but ICE preferred direct".+ // Names only, never the credential itself.+ if (entry.p2pPolicy) {+ const p = entry.p2pPolicy+ console.log(`[p2p] policy · machine=${sid(machineId)} stun=${p.stunUrls.length}`+ + ` turn=${p.turn ? `${p.turn.urls.length} urls` : 'NONE'} openWait=${p.openWaitMs}ms`)+ warmStunUrls(p.stunUrls)+ }try { ws.send(JSON.stringify(crypto.helloFrame())) } catch { /* the close handler below rejects */ }return}@@ -248,8 +259,17 @@ export class RemoteRelayPool {if (frame.type === 'e2e_welcome') {const ok = crypto.handleWelcome((frame.payload ?? {}) as Record<string, unknown>)if (!ok) { if (!settled) { settled = true; clearTimeout(timeout); reject(new RelayConnectError('E2EE_WELCOME_INVALID')) } ; return }- if (entry.p2pPolicy && crypto.terminalP2pVersion === TERMINAL_P2P_PROTOCOL_VERSION) {- this.startP2p(entry)+ // Three ways p2p never even starts, and until now all three looked identical from outside —+ // the terminal just quietly stayed on the ws relay. The peer-version case is the important+ // one: a machine whose CLI predates p2p answers no offer, so NO amount of STUN or TURN can+ // help it. That is a very different problem from "ICE tried and failed".+ if (!entry.p2pPolicy) {+ console.log(`[p2p] off · machine=${sid(machineId)} backend sent no policy (rollout or kill switch)`)+ } else if (crypto.terminalP2pVersion !== TERMINAL_P2P_PROTOCOL_VERSION) {+ console.log(`[p2p] off · machine=${sid(machineId)} peer speaks p2p v${crypto.terminalP2pVersion},`+ + ` we speak v${TERMINAL_P2P_PROTOCOL_VERSION} — no data channel is possible, ws relay only`)+ } else {+ this.startP2p(machineId, entry)}if (!settled) { settled = true; clearTimeout(timeout); resolve() }return@@ -379,7 +399,7 @@ export class RemoteRelayPool {}}- private startP2p(entry: Entry): void {+ private startP2p(machineId: string, entry: Entry): void {const policy = entry.p2pPolicyif (!policy || entry.p2p) returnlet wasDirect = false@@ -390,13 +410,24 @@ export class RemoteRelayPool {try { entry.ws.send(JSON.stringify(wrapped)) } catch { /* relay close handles cleanup */ }},onData: (data) => this.handleP2pData(entry, data),+ // Milestones, so a slow setup can be attributed instead of guessed at: our own gather runs from+ // start to `offer-sent` (setLocalDescription awaits gathering — nothing here trickles), the peer's+ // gather plus one relay round trip lands on `answer-in`, and everything after that is ICE+ // connectivity checks. A 10s failure looks completely different depending on which gap owns it.+ onStep: (step, elapsedMs) => {+ console.log(`[p2p] step · machine=${sid(machineId)} ${step} +${Math.round(elapsedMs)}ms`)+ },onState: (state, setupMs, reason) => {if (state === 'direct') {wasDirect = true// 'relayed' distinguishes a Cloudflare TURN path from a truly direct one. Both are "p2p" as far// as the terminal is concerned, but only one of them is billed per GB.- this.reportP2pResult(entry, 'direct', setupMs, p2p.transport === 'relay' ? 'relayed' : undefined)+ const relayed = p2p.transport === 'relay'+ console.log(`[p2p] connected · machine=${sid(machineId)} via=${relayed ? 'turn' : 'direct'} setup=${Math.round(setupMs)}ms`)+ this.reportP2pResult(entry, 'direct', setupMs, relayed ? 'relayed' : undefined)} else if (state === 'failed' && !wasDirect) {+ console.log(`[p2p] gave up · machine=${sid(machineId)} reason=${reason ?? 'unknown'} after=${Math.round(setupMs)}ms`+ + ` · ${p2p.negotiationDetail} — terminals stay on the ws relay`)this.reportP2pResult(entry, reason === 'negotiation_timeout' ? 'timeout' : 'failed', setupMs, reason)}},@@ -425,6 +456,25 @@ export class RemoteRelayPool {this.noteTerminalResponse(entry, plain, 'p2p')}+ /**+ * Which of the three paths this stream's bytes are on, for the local app's badge.+ *+ * 'p2p' — ICE nominated a direct candidate pair.+ * 'turn' — there IS a data channel, but ICE could only nominate a relay pair, so every byte goes+ * through Cloudflare TURN. Still WebRTC, still E2EE, but billed per GB.+ * 'relay' — no data channel at all; the bytes are riding the backend WebSocket.+ *+ * 'turn' is additive: 'relay' keeps the exact meaning it has always had, so an older Desktop simply+ * drops the unknown value and shows no badge rather than mislabelling a TURN session as a WS one.+ *+ * A null `transport` (channel open but werift exposed no candidate pair yet) reads as 'p2p' — the+ * same optimistic answer this frame already gave before TURN existed.+ */+ private linkMode(entry: Entry, streamId: string): 'p2p' | 'turn' | 'relay' {+ if (!entry.p2pStreams.has(streamId)) return 'relay'+ return entry.p2p?.transport === 'relay' ? 'turn' : 'p2p'+ }+private noteTerminalResponse(entry: Entry, frame: Frame, transport: 'p2p' | 'relay'): void {const payload = framePayload(frame)const streamId = typeof payload.streamId === 'string' ? payload.streamId : ''@@ -435,7 +485,7 @@ export class RemoteRelayPool {// entry.p2pStreams' own post-update membership (the routing source of truth just above), not a// naive echo of `transport`, so a stale/duplicate terminal_ready can never report a mode that// doesn't match what's actually routing.- entry.sink?.sendFrame({ type: 'terminal_link_mode', payload: { streamId, mode: entry.p2pStreams.has(streamId) ? 'p2p' : 'relay' } })+ entry.sink?.sendFrame({ type: 'terminal_link_mode', payload: { streamId, mode: this.linkMode(entry, streamId) } })} else if (frame.type === 'terminal_error' && requestId) {entry.p2pPendingOpens.delete(requestId)} else if (frame.type === 'terminal_closed' && streamId) {diff --git a/cli/src/lib/terminalP2p.spec.ts b/cli/src/lib/terminalP2p.spec.tsindex b8d06973b..7804bcaf0 100644--- a/cli/src/lib/terminalP2p.spec.ts+++ b/cli/src/lib/terminalP2p.spec.ts@@ -1,6 +1,8 @@import { describe, expect, it, vi } from 'vitest'import {+ isRelayedPair,readTurn,+ TERMINAL_P2P_NEGOTIATION_TIMEOUT_MS,TerminalP2pInitiator,TerminalP2pResponderPool,type TerminalP2pSignal,@@ -125,7 +127,7 @@ describe('terminal WebRTC data channel', () => {try {initiator.start()- await vi.advanceTimersByTimeAsync(10_001)+ await vi.advanceTimersByTimeAsync(TERMINAL_P2P_NEGOTIATION_TIMEOUT_MS + 1)expect(states).toContainEqual({ state: 'failed', reason: 'negotiation_timeout' })} finally {vi.useRealTimers()@@ -217,4 +219,29 @@ describe('terminal WebRTC data channel', () => {await responder.stop()}}, 15_000)++ // Real candidate lines, captured from a forced relay-only run against Cloudflare.+ const RELAY = 'candidate:856fe30cc 1 udp 16777215 104.30.136.14 29000 typ relay raddr 14.161.43.75 rport 55397'+ const SRFLX = 'candidate:2b1f0a4c9 1 udp 1686052607 14.161.43.75 55397 typ srflx raddr 192.168.1.16 rport 55397'+ const HOST = 'candidate:9d3e77bb1 1 udp 2130706431 192.168.1.16 55397 typ host'++ it('counts a pair as relayed when EITHER end is a turn allocation', () => {+ // The half that used to be missed: our side is srflx, the peer allocated the relay, and every byte+ // still crosses Cloudflare — reading only the local candidate reported that as direct and+ // under-counted the traffic that gets billed.+ expect(isRelayedPair(SRFLX, RELAY)).toBe(true)+ expect(isRelayedPair(RELAY, SRFLX)).toBe(true)+ expect(isRelayedPair(RELAY, RELAY)).toBe(true)+ })++ it('counts a pair as direct only when neither end relays', () => {+ expect(isRelayedPair(HOST, HOST)).toBe(false)+ expect(isRelayedPair(SRFLX, SRFLX)).toBe(false)+ expect(isRelayedPair(HOST, SRFLX)).toBe(false)+ expect(isRelayedPair(SRFLX)).toBe(false) // remote unknown+ })++ it('does not mistake a host address that merely contains the word', () => {+ expect(isRelayedPair('candidate:1 1 udp 1 10.0.0.1 1 typ host raddr relay.example')).toBe(false)+ })})diff --git a/cli/src/lib/terminalP2p.ts b/cli/src/lib/terminalP2p.tsindex b5cb598e4..e3719b0d2 100644--- a/cli/src/lib/terminalP2p.ts+++ b/cli/src/lib/terminalP2p.ts@@ -1,10 +1,25 @@import { randomUUID } from 'node:crypto'import { RTCPeerConnection, type RTCDataChannel, type RTCIceCandidateInit } from 'werift'+import { env } from '../config/env.js'import { pickTurnUrl, selectStunUrls, type StunSelector } from './stunSelect.js'export const TERMINAL_P2P_PROTOCOL_VERSION = 1export const TERMINAL_P2P_CHANNEL = 'terminal-v1'-export const TERMINAL_P2P_NEGOTIATION_TIMEOUT_MS = 10_000+/**+ * The budget covers BOTH peers' gathering, not just ours, and 10s was too tight for that.+ *+ * Measured: our own gather is 100-400ms, but a peer whose network drops UDP to the STUN server stalls+ * the full 5s werift gather timeout (`getCandidatePromises(addresses, timeout = 5)`, hard-coded, not+ * exposed through RTCPeerConnection) before it can answer — 5027ms with a dead STUN in the list versus+ * 111ms with a live one. Add a TURN allocation over TLS and one relay round trip and the answer lands+ * around 7.2s, leaving under 3s for relay-to-relay connectivity checks, which is not enough. Observed+ * exactly that: `answer-in +7208ms` then death at +10001ms still in `connecting`.+ *+ * Raising it costs nothing user-visible: a terminal that opens before the channel is ready falls back+ * to the ws relay after `openWaitMs` (2.5s) regardless, and negotiation continuing in the background+ * only means LATER opens get p2p. Whichever side fires first aborts for both, so both ends need this.+ */+export const TERMINAL_P2P_NEGOTIATION_TIMEOUT_MS = 25_000export const TERMINAL_P2P_MAX_BUFFERED_BYTES = 2 * 1024 * 1024export const TERMINAL_P2P_SIGNAL_TYPES = new Set([@@ -65,6 +80,8 @@ export interface TerminalP2pInitiatorDeps {onUnavailable?: (reason: string) => voidnow?: () => numberselectStunUrls?: StunSelector+ /** Coarse negotiation milestones, for working out WHERE a slow setup spends its time. */+ onStep?: (step: string, elapsedMs: number) => void}type ReadyWaiter = (ready: boolean) => void@@ -93,6 +110,10 @@ function peerConfig(stunUrls: string[], turn?: TurnChoice): ConstructorParameterif (turn) iceServers.push({ urls: [turn.url], username: turn.username, credential: turn.credential })return {iceServers,+ // Diagnostic only (TERMINAL_P2P_FORCE_RELAY): drops host and srflx so nothing but a TURN+ // allocation can be nominated. Note it also skips STUN gathering entirely inside werift, which is+ // why a forced run is fast even on a network where STUN is unreachable.+ ...(env.TERMINAL_P2P_FORCE_RELAY ? { iceTransportPolicy: 'relay' as const } : {}),// Left at the default 'all' deliberately: that is what produces "direct first, relay only if// nothing else works". werift scores candidates host 126 > srflx 100 > relay 0, so a TURN relay// pair is only ever nominated once every direct pair has failed. Setting 'relay' here would force@@ -123,6 +144,20 @@ function turnChoice(turn: TerminalP2pTurn | undefined, udpReachable: boolean | nreturn url ? { url, username: turn.username, credential: turn.credential } : undefined}+/**+ * Does the nominated pair send its bytes through a TURN allocation?+ *+ * BOTH ends decide this, not just ours. One relay candidate is enough for ICE to connect, so a pair of+ * our srflx with the peer's relay is fully relayed by Cloudflare — and reading only the local side+ * (which is what this did at first) reports it as 'direct' and under-counts exactly the traffic that+ * gets billed per GB. werift's RTCIceCandidate carries only the SDP line, so the type comes out of+ * the string.+ */+export function isRelayedPair(local: string, remote?: string): boolean {+ const relay = /\btyp relay\b/+ return relay.test(local) || (typeof remote === 'string' && relay.test(remote))+}+function channelCanSend(channel: RTCDataChannel | null): channel is RTCDataChannel {return channel?.readyState === 'open'&& channel.bufferedAmount < TERMINAL_P2P_MAX_BUFFERED_BYTES@@ -141,6 +176,7 @@ export class TerminalP2pInitiator {private finished = falseprivate timeout: ReturnType<typeof setTimeout> | null = nullprivate waiters: ReadyWaiter[] = []+ private sawAnswer = falseconstructor(private readonly deps: TerminalP2pInitiatorDeps) {this.now = deps.now ?? (() => Date.now())@@ -148,6 +184,10 @@ export class TerminalP2pInitiator {this.startedAt = this.now()}+ private step(label: string): void {+ this.deps.onStep?.(label, this.now() - this.startedAt)+ }+get isReady(): boolean { return this.ready && channelCanSend(this.channel) }/**@@ -155,13 +195,30 @@ export class TerminalP2pInitiator {* through Cloudflare TURN, which is billed per GB — without reporting this we would have no idea how* much of the traffic costs money. `null` while still negotiating, or if werift exposes no pair.*/+ /**+ * Why a negotiation went nowhere, in one line. The distinction that matters is `answer=no`: it means+ * the peer never replied to our offer, so no candidate of ours — STUN or TURN — was ever going to be+ * tried. That is a peer problem, not a connectivity one, and no amount of relay fixes it.+ */+ get negotiationDetail(): string {+ // Both sides gather before signalling (nothing subscribes to onIceCandidate), so the candidate+ // types live in the descriptions, NOT in the trickle frames — counting those would always read 0+ // and hide exactly the asymmetry that matters: whether the PEER managed a relay candidate too.+ const types = (sdp: string): string =>+ [...new Set([...sdp.matchAll(/typ (\w+)/g)].map((m) => m[1]))].join('/') || 'none'+ return `answer=${this.sawAnswer ? 'yes' : 'no'}`+ + ` ours=${types(this.pc?.localDescription?.sdp ?? '')}`+ + ` peer=${types(this.pc?.remoteDescription?.sdp ?? '')}`+ + ` ice=${this.pc?.connectionState ?? '-'}`+ }+get transport(): 'direct' | 'relay' | null {const pc = this.pcif (!pc || !this.ready) return nullfor (const iceTransport of pc.iceTransports) {- // werift's RTCIceCandidate carries only the SDP line, so the type comes out of the string.- const line = iceTransport.getSelectedCandidatePair?.()?.local?.candidate- if (typeof line === 'string' && line) return /\btyp relay\b/.test(line) ? 'relay' : 'direct'+ const pair = iceTransport.getSelectedCandidatePair?.()+ if (!pair?.local?.candidate) continue+ return isRelayedPair(pair.local.candidate, pair.remote?.candidate) ? 'relay' : 'direct'}return null}@@ -191,6 +248,7 @@ export class TerminalP2pInitiator {// stop()/fail() may have run while the race was in flight. Building the peer connection now would// strand it: nothing holds a reference any more, so its UDP sockets would never be closed.if (this.finished) return+ this.step('stun-raced')const pc = new RTCPeerConnection(peerConfig(stunUrls, turnChoice(this.deps.policy.turn, udpReachable)))const channel = pc.createDataChannel(TERMINAL_P2P_CHANNEL, { ordered: true })this.pc = pc@@ -216,6 +274,7 @@ export class TerminalP2pInitiator {stunUrls: this.deps.policy.stunUrls,...(this.deps.policy.turn ? { turn: this.deps.policy.turn } : {}),})+ this.step('offer-sent')} catch {this.fail('offer_failed')}@@ -229,6 +288,8 @@ export class TerminalP2pInitiator {if (!pc) return truetry {if (type === 'p2p_answer' && typeof payload.sdp === 'string') {+ this.sawAnswer = true+ this.step('answer-in')await pc.setRemoteDescription({ type: 'answer', sdp: payload.sdp })} else if (type === 'p2p_ice_candidate') {await pc.addIceCandidate(payload.candidate ?? null)@@ -298,6 +359,7 @@ export class TerminalP2pInitiator {private wirePeer(pc: RTCPeerConnection): void {pc.connectionStateChange.subscribe((state) => {if (this.pc !== pc || this.finished) return+ this.step(`ice-${state}`)if (state === 'failed' || state === 'disconnected') this.fail(`peer_${state}`)})}