The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
feat: implement TURN-to-direct P2P connection upgrade support with shadow session orchestration
0xken15e49df0690cc7
diff --git a/cli/src/lib/remoteRelay.spec.ts b/cli/src/lib/remoteRelay.spec.tsindex c368473be..689804e66 100644--- a/cli/src/lib/remoteRelay.spec.ts+++ b/cli/src/lib/remoteRelay.spec.ts@@ -23,6 +23,13 @@ function fakeEntry(overrides: Partial<Record<string, unknown>> = {}) {p2pRetryCount: 0,p2pRetryLifetimeTotal: 0,p2pRetryTimer: null,+ upgradeAttempts: 0,+ upgradeTimer: null,+ upgradeShadow: null,+ upgradeDraining: new Map<string, number>(),+ upgradeWaitResolve: null as (() => void) | null,+ upgradeOrphan: null,+ upgradeDone: false,...overrides,}}@@ -284,3 +291,162 @@ describe('RemoteRelayPool live-migration of already-open streams onto p2p', () =expect(entry.sink.sendFrame).not.toHaveBeenCalled()})})++/** Minimal fake of the TerminalP2pInitiator surface promoteToDirect()/scheduleUpgradeAttempt() touch —+ * a real one requires a live WebRTC negotiation, exactly why startP2p() itself has no dedicated test at+ * this level either; this mirrors that existing limit rather than trying to route around it. */+function fakeP2p(overrides: Partial<Record<string, unknown>> = {}) {+ return {+ sessionId: 'shadow-session',+ isReady: true,+ transport: 'direct',+ send: vi.fn(() => true),+ stop: vi.fn(async () => {}),+ ...overrides,+ }+}++describe('RemoteRelayPool TURN-to-direct upgrade', () => {+ const pool = new RemoteRelayPool(+ { accessToken: async () => 'unused' } as never,+ 'ws://unused',+ { pub: new Uint8Array(), priv: new Uint8Array() } as never,+ { pin: () => {}, get: () => null } as never,+ ) as unknown as {+ scheduleUpgradeAttempt: (machineId: string, entry: ReturnType<typeof fakeEntry>) => void+ finishUpgradeAttempt: (machineId: string, entry: ReturnType<typeof fakeEntry>) => void+ promoteToDirect: (machineId: string, entry: ReturnType<typeof fakeEntry>, shadow: ReturnType<typeof fakeP2p>) => Promise<void>+ demoteP2p: (machineId: string, entry: ReturnType<typeof fakeEntry>, reason: string) => void+ // promoteToDirect() re-checks `this.entries.get(machineId) === entry` after every await (the same+ // liveness guard scheduleP2pRetry's timer callback uses) — these tests register the fake entry here+ // directly rather than going through a real dial(), the same shortcut fakeEntry() already takes.+ entries: Map<string, ReturnType<typeof fakeEntry>>+ }++ it('schedules up to 3 upgrade attempts, then gives up for good', () => {+ vi.useFakeTimers()+ const UPGRADE_MAX = 3 // mirrors the unexported P2P_UPGRADE_MAX_ATTEMPTS constant+ try {+ const entry = fakeEntry()+ for (let i = 1; i <= UPGRADE_MAX; i++) {+ pool.scheduleUpgradeAttempt('machine-1', entry)+ expect(entry.upgradeTimer, `attempt #${i} should have been scheduled`).not.toBeNull()+ vi.advanceTimersByTime(60_000)+ entry.upgradeTimer = null // the real callback nulls this before calling attemptUpgrade+ // attemptUpgrade() itself needs a real TerminalP2pInitiator to run — simulate what it does on a+ // failed trial (finishUpgradeAttempt is what decides whether to reschedule or give up).+ entry.upgradeAttempts = i+ pool.finishUpgradeAttempt('machine-1', entry)+ }+ expect(entry.upgradeDone).toBe(true)+ pool.scheduleUpgradeAttempt('machine-1', entry) // refused — done for good+ expect(entry.upgradeTimer).toBeNull()+ } finally {+ vi.useRealTimers()+ }+ })++ it('does not schedule once upgradeDone, even with attempts left', () => {+ const entry = fakeEntry({ upgradeDone: true, upgradeAttempts: 1 })+ pool.scheduleUpgradeAttempt('machine-1', entry)+ expect(entry.upgradeTimer).toBeNull()+ })++ it('cuts over immediately when there are no open streams to drain', async () => {+ const old = fakeP2p({ sessionId: 'old-session' })+ const shadow = fakeP2p({ sessionId: 'shadow-session' })+ const entry = fakeEntry({ p2p: old, upgradeShadow: shadow })+ pool.entries.set('machine-1', entry)++ const promoted = pool.promoteToDirect('machine-1', entry, shadow)+ // No streams means no drain wait — only the promote-ack wait is outstanding.+ expect(entry.upgradeWaitResolve).not.toBeNull()+ expect(entry.p2p).toBe(shadow) // cutover already happened, ack notwithstanding+ entry.upgradeWaitResolve!()+ await promoted++ expect(old.stop).toHaveBeenCalledWith('upgraded', false)+ expect(entry.upgradeDone).toBe(true)+ expect(entry.upgradeOrphan).toBeNull()+ })++ it('drains open streams over the OLD connection before cutting over', async () => {+ const old = fakeP2p({ sessionId: 'old-session' })+ const shadow = fakeP2p({ sessionId: 'shadow-session' })+ const entry = fakeEntry({ p2p: old, upgradeShadow: shadow, p2pStreams: new Set(['s1']) })+ pool.entries.set('machine-1', entry)++ const promoted = pool.promoteToDirect('machine-1', entry, shadow)+ // Drain resync went out over the OLD connection — cutover has NOT happened yet.+ expect(old.send).toHaveBeenCalledWith(JSON.stringify({ type: 'terminal_resync', payload: { streamId: 's1' } }))+ expect(entry.p2p).toBe(old)+ expect(entry.upgradeDraining.has('s1')).toBe(true)++ entry.upgradeWaitResolve!() // simulate the drain-confirmation keyframe arriving+ await Promise.resolve() // let promoteToDirect's await settle onto phase 2+ await Promise.resolve()++ expect(entry.p2p).toBe(shadow) // NOW cut over+ // Phase 2's resync for 's1' rides sessionFor() again — which now routes through entry.p2p (=shadow),+ // since p2pStreams still names 's1' and was never touched by the cutover.+ expect(shadow.send).toHaveBeenCalledWith(JSON.stringify({ type: 'terminal_resync', payload: { streamId: 's1' } }))+ entry.upgradeWaitResolve!() // simulate the promote ack+ await promoted++ expect(old.stop).toHaveBeenCalledWith('upgraded', false)+ })++ it('a drain timeout leaves the old connection completely untouched', async () => {+ vi.useFakeTimers()+ try {+ const old = fakeP2p({ sessionId: 'old-session' })+ const shadow = fakeP2p({ sessionId: 'shadow-session' })+ const entry = fakeEntry({ p2p: old, upgradeShadow: shadow, p2pStreams: new Set(['s1']) })+ pool.entries.set('machine-1', entry)++ const promoted = pool.promoteToDirect('machine-1', entry, shadow)+ await vi.advanceTimersByTimeAsync(5_000)+ await promoted++ expect(entry.p2p).toBe(old) // never cut over+ expect(old.stop).not.toHaveBeenCalled()+ expect(shadow.stop).toHaveBeenCalledWith('upgrade_drain_timeout', true)+ expect(entry.upgradeShadow).toBeNull()+ expect(entry.upgradeDraining.size).toBe(0)+ } finally {+ vi.useRealTimers()+ }+ })++ it('a promote-ack timeout keeps the old connection alive as an orphan rather than guessing it is safe to close', async () => {+ vi.useFakeTimers()+ try {+ const old = fakeP2p({ sessionId: 'old-session' })+ const shadow = fakeP2p({ sessionId: 'shadow-session' })+ const entry = fakeEntry({ p2p: old, upgradeShadow: shadow })+ pool.entries.set('machine-1', entry)++ const promoted = pool.promoteToDirect('machine-1', entry, shadow)+ await vi.advanceTimersByTimeAsync(5_000)+ await promoted++ expect(entry.p2p).toBe(shadow) // already cut over — no reason to roll back a proven-direct pair+ expect(old.stop).not.toHaveBeenCalled() // NOT guessed closed+ expect(entry.upgradeOrphan).toBe(old) // left for entry-teardown cleanup instead+ expect(entry.upgradeDone).toBe(true)+ } finally {+ vi.useRealTimers()+ }+ })++ it('demoteP2p cancels an in-flight upgrade trial without touching the primary retry policy', () => {+ const shadow = fakeP2p({ sessionId: 'shadow-session' })+ const entry = fakeEntry({ upgradeShadow: shadow, upgradeTimer: setTimeout(() => {}, 60_000) })++ pool.demoteP2p('machine-1', entry, 'send_failed')++ expect(shadow.stop).toHaveBeenCalledWith('primary_demoted', true)+ expect(entry.upgradeShadow).toBeNull()+ expect(entry.upgradeTimer).toBeNull()+ })+})diff --git a/cli/src/lib/remoteRelay.ts b/cli/src/lib/remoteRelay.tsindex 0027c8c42..652c6fba1 100644--- a/cli/src/lib/remoteRelay.ts+++ b/cli/src/lib/remoteRelay.ts@@ -53,6 +53,22 @@ const P2P_RETRY_LIFETIME_CAP = 10// on it and lets the ordinary demote-on-mismatch rule apply again.const P2P_MIGRATION_TTL_MS = 30_000+// A connection that reached 'direct' only via a Cloudflare TURN relay pair still costs per GB — worth+// trying, periodically, for a truly direct pair instead. Unlike p2pRetry* above this NEVER touches the+// live connection until a replacement has already proven itself: see scheduleUpgradeAttempt() and+// promoteToDirect(). Capped at 3 tries and never reset (no success to reset it FOR — reaching direct+// ends the attempts for good, and 3 relay results in a row is treated as "this path is what it is").+const P2P_UPGRADE_MAX_ATTEMPTS = 3+const P2P_UPGRADE_RETRY_DELAY_MS = 60_000+// Shorter than TERMINAL_P2P_NEGOTIATION_TIMEOUT_MS's 25s: the network has already proven itself capable+// of a full negotiation (that's how the current TURN connection exists at all), so this trial doesn't+// need the first-contact worst-case budget.+const P2P_UPGRADE_ATTEMPT_TIMEOUT_MS = 15_000+// Both phases of the cutover (drain confirmation, then promote ack) wait on one round trip over an+// already-live channel — 5s is generous for that, not for a cold negotiation.+const P2P_UPGRADE_DRAIN_TIMEOUT_MS = 5_000+const P2P_PROMOTE_ACK_TIMEOUT_MS = 5_000+export class RelayConnectError extends Error {constructor(message: string, readonly closeCode?: number) {super(message)@@ -90,6 +106,26 @@ interface Entry {* retrying forever purely because each success resets p2pRetryCount. */p2pRetryLifetimeTotal: numberp2pRetryTimer: ReturnType<typeof setTimeout> | null+ /** TURN-to-direct upgrade trials — see scheduleUpgradeAttempt()/attemptUpgrade()/promoteToDirect().+ * Entirely separate bookkeeping from p2pMigrating/p2pRetry*: this never touches entry.p2p (the live,+ * working connection) until a replacement has already proven itself direct AND drained. */+ upgradeAttempts: number+ upgradeTimer: ReturnType<typeof setTimeout> | null+ upgradeShadow: TerminalP2pInitiator | null+ /** streamId -> when its drain-barrier resync went out, RIGHT before a cutover — distinct from+ * p2pMigrating (that one is for WS->p2p; this is for p2p(turn)->p2p(direct), and the confirmation for+ * each arrives over a different transport, so sharing one map would let the two migrations cross-talk. */+ upgradeDraining: Map<string, number>+ /** Resolves whichever of the two sequential waits (drain-complete, then promote-ack) is currently+ * outstanding — the two never overlap, so one field is enough. */+ upgradeWaitResolve: (() => void) | null+ /** Set only when a promote_ack never arrived: the old connection was intentionally left open rather+ * than guessed closed (see promoteToDirect()) — cleaned up wherever the entry itself is torn down. */+ upgradeOrphan: TerminalP2pInitiator | null+ /** True once this entry has either cut over to a true direct pair, or spent all 3 attempts without+ * one — either way, stop trying. Never reset (unlike p2pRetryCount): there is no "next demote" to+ * re-arm a budget for here, since staying on direct or giving up are both permanent for this entry. */+ upgradeDone: boolean}export interface RelaySession {@@ -148,6 +184,9 @@ export class RemoteRelayPool {if (entry.heartbeatTimer) clearInterval(entry.heartbeatTimer)if (entry.lingerTimer) clearTimeout(entry.lingerTimer)if (entry.p2pRetryTimer) clearTimeout(entry.p2pRetryTimer)+ if (entry.upgradeTimer) clearTimeout(entry.upgradeTimer)+ void entry.upgradeShadow?.stop('invalidated', false)+ void entry.upgradeOrphan?.stop('invalidated', false)// The caller is invalidating so it can immediately acquire() a fresh entry on the SAME local// connection (it just got a forceReconnect select) — null this out first so the generic// `ws.on('close', ...)` cleanup below doesn't turn around and close that same local socket via a@@ -232,6 +271,13 @@ export class RemoteRelayPool {p2pRetryCount: 0,p2pRetryLifetimeTotal: 0,p2pRetryTimer: null,+ upgradeAttempts: 0,+ upgradeTimer: null,+ upgradeShadow: null,+ upgradeDraining: new Map(),+ upgradeWaitResolve: null,+ upgradeOrphan: null,+ upgradeDone: false,}// Two phases before this connection is usable: (1) machine_select ack, (2) this daemon's own// e2e_hello/e2e_welcome as the "client" role — see lib/e2ee/relayClient.ts. Only once BOTH are done@@ -357,7 +403,24 @@ export class RemoteRelayPool {if (!plain) returnconst type = typeof plain.type === 'string' ? plain.type : ''if (TERMINAL_P2P_SIGNAL_TYPES.has(type)) {- void entry.p2p?.handleSignal(type, plain.payload)+ // p2p_promote/p2p_promote_ack belong to the upgrade orchestration below, not to+ // TerminalP2pInitiator's own protocol — it would just silently no-op on them.+ if (type === 'p2p_promote_ack') {+ const ackPayload = plain.payload as { sessionId?: unknown } | undefined+ if (typeof ackPayload?.sessionId === 'string' && ackPayload.sessionId === entry.p2p?.sessionId) {+ entry.upgradeWaitResolve?.()+ }+ return+ }+ // A shadow trial negotiates its OWN session alongside entry.p2p (the live primary) — route by+ // sessionId so its answer/candidates/abort reach it instead of being checked against (and+ // silently dropped by) the primary's handleSignal, which only matches its own sessionId.+ const signalSessionId = (plain.payload as { sessionId?: unknown } | undefined)?.sessionId+ if (entry.upgradeShadow && typeof signalSessionId === 'string' && signalSessionId === entry.upgradeShadow.sessionId) {+ void entry.upgradeShadow.handleSignal(type, plain.payload)+ } else {+ void entry.p2p?.handleSignal(type, plain.payload)+ }return}// Forward the real frame FIRST: for `terminal_ready`, the Desktop app's TerminalSession learns@@ -382,6 +445,9 @@ export class RemoteRelayPool {ws.on('close', (code, reasonBuf) => {if (entry.heartbeatTimer) clearInterval(entry.heartbeatTimer)if (entry.p2pRetryTimer) clearTimeout(entry.p2pRetryTimer)+ if (entry.upgradeTimer) clearTimeout(entry.upgradeTimer)+ void entry.upgradeShadow?.stop('relay_closed', false)+ void entry.upgradeOrphan?.stop('relay_closed', false)void entry.p2p?.stop('relay_closed', false)entry.p2p = nullthis.entries.delete(machineId)@@ -449,6 +515,9 @@ export class RemoteRelayPool {entry.lingerTimer = setTimeout(() => {if (!entry.sink) {if (entry.p2pRetryTimer) clearTimeout(entry.p2pRetryTimer)+ if (entry.upgradeTimer) clearTimeout(entry.upgradeTimer)+ void entry.upgradeShadow?.stop('idle', false)+ void entry.upgradeOrphan?.stop('idle', false)void entry.p2p?.stop('idle', false)entry.p2p = nulltry { entry.ws.close(1000, 'idle') } catch { /* ignore */ }@@ -488,6 +557,7 @@ export class RemoteRelayPool {console.log(`[p2p] connected · machine=${sid(machineId)} via=${relayed ? 'turn' : 'direct'} setup=${Math.round(setupMs)}ms`)this.reportP2pResult(entry, 'direct', setupMs, relayed ? 'relayed' : undefined)this.promoteOpenStreams(machineId, entry)+ if (relayed && !entry.upgradeDone) this.scheduleUpgradeAttempt(machineId, entry)} else if (state === 'failed' && !wasDirect) {console.log(`[p2p] gave up · machine=${sid(machineId)} reason=${reason ?? 'unknown'} after=${Math.round(setupMs)}ms`+ ` · ${p2p.negotiationDetail} — terminals stay on the ws relay`)@@ -515,6 +585,12 @@ export class RemoteRelayPool {// stream's bytes are now genuinely arriving over p2p — safe to re-arm the ordinary// demote-on-mismatch rule for it (see the guard in the ws binary handler above).entry.p2pMigrating.delete(clear.streamId)+ // Drain-barrier confirmation for a TURN-to-direct upgrade in flight (promoteToDirect() phase 1):+ // this keyframe is the OLD (still-primary) connection's answer to the resync it sent to itself+ // over p2p, proving everything before that point has been accounted for.+ if (entry.upgradeDraining.delete(clear.streamId) && entry.upgradeDraining.size === 0) {+ entry.upgradeWaitResolve?.()+ }const local = encodeTerminalLocal(clear)if (local) entry.sink?.sendBinary(local)return@@ -579,6 +655,12 @@ export class RemoteRelayPool {}private demoteP2p(machineId: string, entry: Entry, reason: string): void {+ // The primary just died (or is dying) — an upgrade trial in flight for it no longer means anything,+ // and scheduleP2pRetry() below is about to take over recovery via the normal path anyway.+ if (entry.upgradeTimer) { clearTimeout(entry.upgradeTimer); entry.upgradeTimer = null }+ if (entry.upgradeShadow) { void entry.upgradeShadow.stop('primary_demoted', true); entry.upgradeShadow = null }+ entry.upgradeDraining.clear()+ entry.upgradeWaitResolve = nullconst p2p = entry.p2pentry.p2p = nullconst streamIds = [...entry.p2pStreams]@@ -656,6 +738,172 @@ export class RemoteRelayPool {// responder committed its own flip. Until then the demote-on-mismatch rule stays suppressed for it.}+ /**+ * Kicks off a periodic attempt to replace a TURN-relayed p2p connection with a true direct one.+ * Never touches entry.p2p until attemptUpgrade()'s shadow has already proven itself — see the Entry+ * interface's doc on upgradeShadow/upgradeDraining.+ */+ private scheduleUpgradeAttempt(machineId: string, entry: Entry): void {+ if (entry.upgradeTimer || entry.upgradeShadow || entry.upgradeDone) return+ if (entry.upgradeAttempts >= P2P_UPGRADE_MAX_ATTEMPTS) { entry.upgradeDone = true; return }+ entry.upgradeTimer = setTimeout(() => {+ entry.upgradeTimer = null+ if (this.entries.get(machineId) !== entry) return // entry was torn down/replaced meanwhile+ this.attemptUpgrade(machineId, entry)+ }, P2P_UPGRADE_RETRY_DELAY_MS)+ entry.upgradeTimer.unref?.()+ }++ /** One trial: negotiate a brand-new (shadow) p2p connection from scratch, entirely independent of the+ * live primary, and see whether IT lands on a direct pair. The primary keeps serving traffic exactly+ * as before for the whole trial — nothing here is wired into entry.p2pStreams/sessionFor() routing+ * until promoteToDirect() has already proven success, so nothing here can disrupt it. */+ private attemptUpgrade(machineId: string, entry: Entry): void {+ if (entry.upgradeDone || !entry.p2p?.isReady || entry.p2p.transport !== 'relay' || !entry.p2pPolicy) return+ entry.upgradeAttempts+++ const policy = entry.p2pPolicy+ const shadow: TerminalP2pInitiator = new TerminalP2pInitiator({+ policy,+ upgrade: true,+ sendSignal: (type, payload) => {+ const wrapped = entry.crypto.wrapOutgoing({ type, payload })+ try { entry.ws.send(JSON.stringify(wrapped)) } catch { /* relay close handles cleanup */ }+ },+ // Identical to the primary's own wiring — correct both before promotion (nothing routes real data+ // here yet, so this never actually fires) and after (once promoted, this IS the primary).+ onData: (data) => this.handleP2pData(entry, data),+ onStep: (step, elapsedMs) => {+ console.log(`[p2p-upgrade] step · machine=${sid(machineId)} attempt=${entry.upgradeAttempts} ${step} +${Math.round(elapsedMs)}ms`)+ },+ onState: (state, setupMs, reason) => {+ console.log(`[p2p-upgrade] ${state} · machine=${sid(machineId)} attempt=${entry.upgradeAttempts}`+ + ` +${Math.round(setupMs)}ms${reason ? ` reason=${reason}` : ''}`)+ },+ onUnavailable: (reason) => {+ if (entry.p2p === shadow) {+ // Already promoted and now failing for real — an ordinary primary failure from here on.+ this.demoteP2p(machineId, entry, reason)+ } else if (entry.upgradeShadow === shadow) {+ // Died as a trial, before cutover — the primary was never touched. Just abandon this attempt.+ entry.upgradeShadow = null+ entry.upgradeDraining.clear()+ entry.upgradeWaitResolve = null+ this.finishUpgradeAttempt(machineId, entry)+ }+ },+ })+ entry.upgradeShadow = shadow+ shadow.start()+ void this.runUpgradeAttempt(machineId, entry, shadow)+ }++ private async runUpgradeAttempt(machineId: string, entry: Entry, shadow: TerminalP2pInitiator): Promise<void> {+ const ok = await shadow.waitUntilReady(P2P_UPGRADE_ATTEMPT_TIMEOUT_MS)+ if (entry.upgradeShadow !== shadow || this.entries.get(machineId) !== entry) return // superseded/torn down meanwhile+ if (!ok || shadow.transport === 'relay') {+ entry.upgradeShadow = null+ void shadow.stop('upgrade_no_gain', true)+ this.finishUpgradeAttempt(machineId, entry)+ return+ }+ await this.promoteToDirect(machineId, entry, shadow)+ }++ private finishUpgradeAttempt(machineId: string, entry: Entry): void {+ if (entry.upgradeAttempts >= P2P_UPGRADE_MAX_ATTEMPTS) {+ entry.upgradeDone = true+ console.log(`[p2p-upgrade] giving up · machine=${sid(machineId)} staying on turn after ${entry.upgradeAttempts} attempts`)+ return+ }+ this.scheduleUpgradeAttempt(machineId, entry)+ }++ /**+ * Cuts entry.p2p over to `shadow`, which has already proven itself a true direct pair. Every bail-out+ * before the actual swap (the `entry.p2p = shadow` line) leaves the primary completely untouched —+ * that line is the one moment this can no longer be undone, and everything before it is written to+ * make that moment as late and as certain as possible.+ */+ private async promoteToDirect(machineId: string, entry: Entry, shadow: TerminalP2pInitiator): Promise<void> {+ const old = entry.p2p+ if (!old || entry.upgradeShadow !== shadow || this.entries.get(machineId) !== entry) {+ void shadow.stop('upgrade_stale', true)+ if (entry.upgradeShadow === shadow) entry.upgradeShadow = null+ this.finishUpgradeAttempt(machineId, entry)+ return+ }+ const streamIds = [...entry.p2pStreams]+ if (streamIds.length > 0) {+ // Phase 1 — drain barrier over the OLD (still-primary, still fully live) connection.+ const drained = await this.waitForUpgradeMilestone(entry, () => {+ for (const streamId of streamIds) entry.upgradeDraining.set(streamId, Date.now())+ for (const streamId of streamIds) {+ void this.sessionFor(machineId, entry).send({ type: 'terminal_resync', payload: { streamId } })+ }+ }, P2P_UPGRADE_DRAIN_TIMEOUT_MS)+ if (entry.upgradeShadow !== shadow || this.entries.get(machineId) !== entry) {+ // Torn down/superseded while draining — old was never touched either way, just clean up.+ entry.upgradeDraining.clear()+ void shadow.stop('upgrade_stale', true)+ return+ }+ if (!drained) {+ // Timed out — OLD IS STILL COMPLETELY UNTOUCHED. Abandon just this attempt.+ entry.upgradeDraining.clear()+ entry.upgradeShadow = null+ void shadow.stop('upgrade_drain_timeout', true)+ this.finishUpgradeAttempt(machineId, entry)+ return+ }+ }+ // Phase 2 — the actual cutover. entry.p2pStreams is untouched: every streamId in it was already+ // p2p and stays p2p, only the object behind entry.p2p changes.+ entry.p2p = shadow+ entry.upgradeShadow = null+ const acked = await this.waitForUpgradeMilestone(entry, () => {+ for (const streamId of streamIds) {+ void this.sessionFor(machineId, entry).send({ type: 'terminal_resync', payload: { streamId } })+ }+ const wrapped = entry.crypto.wrapOutgoing({+ type: 'p2p_promote',+ payload: { sessionId: shadow.sessionId, protocolVersion: TERMINAL_P2P_PROTOCOL_VERSION },+ })+ try { entry.ws.send(JSON.stringify(wrapped)) } catch { /* relay close handles cleanup */ }+ }, P2P_PROMOTE_ACK_TIMEOUT_MS)+ entry.upgradeDone = true // either way, this entry is done trying — see the field's own doc comment+ if (acked) {+ void old.stop('upgraded', false)+ console.log(`[p2p-upgrade] promoted · machine=${sid(machineId)} attempt=${entry.upgradeAttempts}`)+ } else {+ // Ack never arrived — keep OLD alive rather than guess it is safe to close; entry.upgradeOrphan is+ // closed wherever the entry itself is torn down (invalidate/ws-close/idle-linger).+ entry.upgradeOrphan = old+ console.log(`[p2p-upgrade] promote ack timeout · machine=${sid(machineId)} — keeping the old connection`+ + ' open until this entry is torn down')+ }+ }++ /** One shared waiter shape for promoteToDirect()'s two sequential phases (drain-complete, then+ * promote-ack) — they never overlap, so entry.upgradeWaitResolve is reused rather than doubled.+ * `start` runs AFTER the timeout/resolver are armed, so a same-tick resolve inside it can never race+ * past a listener that is not wired up yet. */+ private waitForUpgradeMilestone(entry: Entry, start: () => void, timeoutMs: number): Promise<boolean> {+ return new Promise<boolean>((resolve) => {+ let settled = false+ const done = (ok: boolean): void => {+ if (settled) return+ settled = true+ clearTimeout(timer)+ entry.upgradeWaitResolve = null+ resolve(ok)+ }+ const timer = setTimeout(() => done(false), timeoutMs)+ timer.unref?.()+ entry.upgradeWaitResolve = () => done(true)+ start()+ })+ }+private reportP2pResult(entry: Entry, outcome: string, setupMs?: number, reason?: string): void {try {entry.ws.send(JSON.stringify({diff --git a/cli/src/lib/terminalP2p.ts b/cli/src/lib/terminalP2p.tsindex 071b1e0bd..aa527a46d 100644--- a/cli/src/lib/terminalP2p.ts+++ b/cli/src/lib/terminalP2p.ts@@ -27,6 +27,12 @@ export const TERMINAL_P2P_SIGNAL_TYPES = new Set(['p2p_answer','p2p_ice_candidate','p2p_abort',+ // Owned entirely by RemoteRelayPool's TURN-to-direct upgrade orchestration (remoteRelay.ts), not by+ // TerminalP2pInitiator/TerminalP2pResponderPool's own protocol — these two just need to ride the same+ // encrypted signaling channel. p2p_promote: "cut the primary over to this already-negotiated shadow+ // session now." p2p_promote_ack: the responder confirming it did.+ 'p2p_promote',+ 'p2p_promote_ack',])export const TERMINAL_P2P_DOWN_TYPES = new Set([@@ -70,6 +76,9 @@ export interface TerminalP2pSignal {* two peers, the credential is short-lived, and the backend minted it in the first place.*/turn?: TerminalP2pTurn+ /** Set on the offer only when this negotiation is a TURN-to-direct upgrade trial for an already-live+ * connId, not a fresh session — see TerminalP2pResponderPool.acceptUpgradeOffer(). */+ upgrade?: true}export interface TerminalP2pInitiatorDeps {@@ -82,6 +91,10 @@ export interface TerminalP2pInitiatorDeps {selectStunUrls?: StunSelector/** Coarse negotiation milestones, for working out WHERE a slow setup spends its time. */onStep?: (step: string, elapsedMs: number) => void+ /** This instance is a shadow trial negotiated alongside an already-live connection (see+ * RemoteRelayPool's TURN-to-direct upgrade), not a fresh session — tags the offer so the responder+ * routes it to acceptUpgradeOffer() instead of tearing down the connId's live entry. */+ upgrade?: boolean}type ReadyWaiter = (ready: boolean) => void@@ -273,6 +286,7 @@ export class TerminalP2pInitiator {// need not agree on a server — a srflx candidate is each peer's own public address.stunUrls: this.deps.policy.stunUrls,...(this.deps.policy.turn ? { turn: this.deps.policy.turn } : {}),+ ...(this.deps.upgrade ? { upgrade: true as const } : {}),})this.step('offer-sent')} catch {@@ -418,6 +432,13 @@ export class TerminalP2pResponderPool {* loser's peer connection would be orphaned by the winner's entries.set(). Bump on entry, re-check* after every await. Pre-existing race — the STUN selection below only widens the window. */private readonly offerSeq = new Map<string, number>()+ /** One live connId's primary connection may have an upgrade trial negotiating alongside it — kept in+ * a separate map, keyed by the SAME connId, so the primary is never touched while a trial is live.+ * See acceptUpgradeOffer()/promote(). */+ private readonly shadowEntries = new Map<string, ResponderEntry>()+ /** Same purpose as offerSeq, but tracked separately: a primary offer and an upgrade offer for the+ * same connId are unrelated negotiations and must not invalidate each other's generation. */+ private readonly shadowOfferSeq = new Map<string, number>()private readonly selectStunUrls: StunSelectorconstructor(private readonly deps: TerminalP2pResponderPoolDeps) {@@ -429,16 +450,33 @@ export class TerminalP2pResponderPool {const payload = parseSignal(value)if (!payload) return trueif (type === 'p2p_offer' && typeof payload.sdp === 'string') {- await this.acceptOffer(connId, payload)+ if (payload.upgrade === true) await this.acceptUpgradeOffer(connId, payload)+ else await this.acceptOffer(connId, payload)+ return true+ }+ if (type === 'p2p_promote') {+ await this.promote(connId, payload)return true}const entry = this.entries.get(connId)- if (!entry || entry.sessionId !== payload.sessionId) return true- try {- if (type === 'p2p_ice_candidate') await entry.pc.addIceCandidate(payload.candidate ?? null)- else if (type === 'p2p_abort') await this.closeConnection(connId, payload.reason || 'peer_aborted', false)- } catch {- await this.closeConnection(connId, 'signal_invalid')+ if (entry && entry.sessionId === payload.sessionId) {+ try {+ if (type === 'p2p_ice_candidate') await entry.pc.addIceCandidate(payload.candidate ?? null)+ else if (type === 'p2p_abort') await this.closeConnection(connId, payload.reason || 'peer_aborted', false)+ } catch {+ await this.closeConnection(connId, 'signal_invalid')+ }+ return true+ }+ // Not the primary's session — try the upgrade trial, if one is in flight for this connId.+ const shadow = this.shadowEntries.get(connId)+ if (shadow && shadow.sessionId === payload.sessionId) {+ try {+ if (type === 'p2p_ice_candidate') await shadow.pc.addIceCandidate(payload.candidate ?? null)+ else if (type === 'p2p_abort') await this.closeShadow(connId, payload.reason || 'peer_aborted')+ } catch {+ await this.closeShadow(connId, 'signal_invalid')+ }}return true}@@ -456,16 +494,13 @@ export class TerminalP2pResponderPool {}}- async closeConnection(connId: string, reason = 'closed', notifyPeer = true): Promise<void> {- const entry = this.entries.get(connId)- if (!entry || entry.closing) return+ /** Shared teardown body for both the primary map and the shadow map — takes the entry directly+ * rather than looking it up by connId, because by the time promote() needs to close the OLD primary,+ * `entries.get(connId)` already points at the just-promoted shadow. Looking it up again here would+ * close the wrong connection. */+ private async teardown(connId: string, entry: ResponderEntry, reason: string, notifyPeer: boolean): Promise<void> {+ if (entry.closing) returnentry.closing = true- this.entries.delete(connId)- // Every reason but 'superseded' retires the connId for good, so drop its generation counter too or- // the map grows for the life of the daemon. 'superseded' is excluded because that call comes from- // acceptOffer itself, which has already claimed the current generation and still needs it as its- // own liveness check across the awaits that follow.- if (reason !== 'superseded') this.offerSeq.delete(connId)clearTimeout(entry.timeout)if (notifyPeer) {this.deps.sendSignal(connId, 'p2p_abort', {@@ -478,11 +513,40 @@ export class TerminalP2pResponderPool {await entry.pc.close().catch(() => { /* best effort */ })}+ async closeConnection(connId: string, reason = 'closed', notifyPeer = true): Promise<void> {+ const entry = this.entries.get(connId)+ if (entry && !entry.closing) {+ this.entries.delete(connId)+ // Every reason but 'superseded' retires the connId for good, so drop its generation counter too+ // or the map grows for the life of the daemon. 'superseded' is excluded because that call comes+ // from acceptOffer itself, which has already claimed the current generation and still needs it as+ // its own liveness check across the awaits that follow.+ if (reason !== 'superseded') this.offerSeq.delete(connId)+ await this.teardown(connId, entry, reason, notifyPeer)+ }+ // A trial upgrade tied to this connId's primary no longer means anything once the primary itself is+ // gone (or is being superseded by a fresh, non-upgrade offer) — retire it too rather than leaking it.+ await this.closeShadow(connId, reason)+ }++ private async closeShadow(connId: string, reason: string): Promise<void> {+ const shadow = this.shadowEntries.get(connId)+ if (!shadow || shadow.closing) return+ this.shadowEntries.delete(connId)+ this.shadowOfferSeq.delete(connId)+ // The initiator tracks its own shadow's lifecycle locally (waitUntilReady's own timeout) rather than+ // depending on a signal from here, so this never notifies the peer — one fewer round trip for a+ // trial that, by definition, has no traffic riding on it yet.+ await this.teardown(connId, shadow, reason, false)+ }+async stop(): Promise<void> {await Promise.all([...this.entries.keys()].map((connId) => this.closeConnection(connId, 'shutdown', false)))+ await Promise.all([...this.shadowEntries.keys()].map((connId) => this.closeShadow(connId, 'shutdown')))// Also drops generations for offers still inside their STUN race, which makes them bail instead of// building a peer connection nothing would ever close.this.offerSeq.clear()+ this.shadowOfferSeq.clear()}private async acceptOffer(connId: string, payload: TerminalP2pSignal): Promise<void> {@@ -490,13 +554,42 @@ export class TerminalP2pResponderPool {this.offerSeq.set(connId, seq)await this.closeConnection(connId, 'superseded', false)if (this.offerSeq.get(connId) !== seq) return+ const entry = await this.buildResponderEntry(connId, payload, () => this.offerSeq.get(connId) === seq)+ if (!entry) return+ this.entries.set(connId, entry)+ this.wireResponderEntry(connId, entry, () => this.entries.get(connId) === entry, (reason) => this.closeConnection(connId, reason))+ await this.answerOffer(connId, entry, payload, () => this.entries.get(connId) === entry, (reason) => this.closeConnection(connId, reason))+ }++ /** Mirrors acceptOffer(), but never touches the connId's live primary entry — see the class-level+ * doc on shadowEntries. Failure at any point here costs nothing but this one trial. */+ private async acceptUpgradeOffer(connId: string, payload: TerminalP2pSignal): Promise<void> {+ const seq = (this.shadowOfferSeq.get(connId) ?? 0) + 1+ this.shadowOfferSeq.set(connId, seq)+ await this.closeShadow(connId, 'superseded') // retire a stale trial for this connId, if any+ if (this.shadowOfferSeq.get(connId) !== seq) return+ const entry = await this.buildResponderEntry(connId, payload, () => this.shadowOfferSeq.get(connId) === seq)+ if (!entry) return+ this.shadowEntries.set(connId, entry)+ this.wireResponderEntry(connId, entry, () => this.shadowEntries.get(connId) === entry, (reason) => this.closeShadow(connId, reason))+ await this.answerOffer(connId, entry, payload, () => this.shadowEntries.get(connId) === entry, (reason) => this.closeShadow(connId, reason))+ }++ /** STUN race + RTCPeerConnection construction shared by acceptOffer/acceptUpgradeOffer. `stillCurrent`+ * is re-checked after the only await in here (the STUN race) so a superseded offer never publishes a+ * peer connection nothing would ever close. Returns null when superseded — caller just returns. */+ private async buildResponderEntry(+ connId: string,+ payload: TerminalP2pSignal,+ stillCurrent: () => boolean,+ ): Promise<ResponderEntry | null> {const offeredStunUrls = Array.isArray(payload.stunUrls)? payload.stunUrls.filter((url): url is string => typeof url === 'string' && /^stuns?:/i.test(url)).slice(0, 8): []// The offerer raced these too, and may well have landed on a different server. That is fine: a// srflx candidate is each peer's own public address, so the two sides need not agree on who to ask.const selection = await this.selectStunUrls(offeredStunUrls)- if (this.offerSeq.get(connId) !== seq) return+ if (!stillCurrent()) return nullconst turn = turnChoice(readTurn(payload.turn), selection.udpReachable)const pc = new RTCPeerConnection(peerConfig(selection.urls, turn))const entry: ResponderEntry = {@@ -506,51 +599,80 @@ export class TerminalP2pResponderPool {ready: false,closing: false,timeout: setTimeout(() => {- this.deps.onUnavailable?.(connId, 'negotiation_timeout')- void this.closeConnection(connId, 'negotiation_timeout')+ if (entry.ready) this.deps.onUnavailable?.(connId, 'negotiation_timeout')+ void (stillCurrent() ? this.closeConnection(connId, 'negotiation_timeout') : this.closeShadow(connId, 'negotiation_timeout'))}, TERMINAL_P2P_NEGOTIATION_TIMEOUT_MS),}entry.timeout.unref?.()- this.entries.set(connId, entry)- pc.connectionStateChange.subscribe((state) => {- if (this.entries.get(connId) !== entry || entry.closing) return+ return entry+ }++ /** connectionStateChange/onDataChannel wiring shared by acceptOffer/acceptUpgradeOffer. `notifyPeer`+ * reporting via deps.onUnavailable is deliberately gated on `isPrimary()` — a trial failing before it+ * is ever promoted is not the connId's p2p becoming unavailable, since the real primary (if any) is+ * untouched throughout. */+ private wireResponderEntry(+ connId: string,+ entry: ResponderEntry,+ isCurrent: () => boolean,+ close: (reason: string) => Promise<void>,+ ): void {+ entry.pc.connectionStateChange.subscribe((state) => {+ if (!isCurrent() || entry.closing) returnif (state === 'failed' || state === 'disconnected') {- if (entry.ready) this.deps.onUnavailable?.(connId, `peer_${state}`)- void this.closeConnection(connId, `peer_${state}`)+ if (entry.ready && this.entries.get(connId) === entry) this.deps.onUnavailable?.(connId, `peer_${state}`)+ void close(`peer_${state}`)}})- pc.onDataChannel.subscribe((channel) => {- if (this.entries.get(connId) !== entry || channel.label !== TERMINAL_P2P_CHANNEL) {+ entry.pc.onDataChannel.subscribe((channel) => {+ if (!isCurrent() || channel.label !== TERMINAL_P2P_CHANNEL) {channel.close()return}entry.channel = channelchannel.bufferedAmountLowThreshold = 256 * 1024channel.stateChanged.subscribe((state) => {- if (this.entries.get(connId) !== entry || entry.closing) return+ if (!isCurrent() || entry.closing) returnif (state === 'open') {entry.ready = trueclearTimeout(entry.timeout)} else if ((state === 'closed' || state === 'closing') && entry.ready) {entry.ready = false- this.deps.onUnavailable?.(connId, 'channel_closed')- void this.closeConnection(connId, 'channel_closed')+ if (this.entries.get(connId) === entry) this.deps.onUnavailable?.(connId, 'channel_closed')+ void close('channel_closed')}})channel.onMessage.subscribe((data) => {- if (this.entries.get(connId) === entry && !entry.closing) this.deps.onData(connId, data)+ // True for the trial's own channel too — never actually reached before promote() moves this+ // entry into `entries`, since the initiator only ever writes to its OWN entry.p2p, which stays+ // the primary throughout a trial. Kept identical to the pre-refactor primary-only check instead+ // of narrowing it, so promotion needs no re-wiring here.+ if ((this.entries.get(connId) === entry || this.shadowEntries.get(connId) === entry) && !entry.closing) {+ this.deps.onData(connId, data)+ }})channel.error.subscribe(() => {- if (entry.ready) this.deps.onUnavailable?.(connId, 'channel_error')- void this.closeConnection(connId, 'channel_error')+ if (entry.ready && this.entries.get(connId) === entry) this.deps.onUnavailable?.(connId, 'channel_error')+ void close('channel_error')})})+ }++ /** setRemoteDescription/createAnswer/setLocalDescription + sending the answer — shared tail of+ * acceptOffer/acceptUpgradeOffer. */+ private async answerOffer(+ connId: string,+ entry: ResponderEntry,+ payload: TerminalP2pSignal,+ isCurrent: () => boolean,+ close: (reason: string) => Promise<void>,+ ): Promise<void> {try {- await pc.setRemoteDescription({ type: 'offer', sdp: payload.sdp! })- const answer = await pc.createAnswer()- await pc.setLocalDescription(answer)- if (this.entries.get(connId) !== entry || entry.closing) return- const local = pc.localDescription+ await entry.pc.setRemoteDescription({ type: 'offer', sdp: payload.sdp! })+ const answer = await entry.pc.createAnswer()+ await entry.pc.setLocalDescription(answer)+ if (!isCurrent() || entry.closing) return+ const local = entry.pc.localDescriptionif (!local) throw new Error('local_description_missing')this.deps.sendSignal(connId, 'p2p_answer', {sessionId: payload.sessionId,@@ -558,7 +680,26 @@ export class TerminalP2pResponderPool {sdp: local.sdp,})} catch {- await this.closeConnection(connId, 'answer_failed')+ await close('answer_failed')}}++ /** Cuts the connId's primary connection over to an already-negotiated-and-ready shadow trial. The+ * shadow must already be at the sessionId named in payload — this never builds a peer connection+ * itself, only swaps which one `entries` points at and retires whichever was there before. */+ private async promote(connId: string, payload: TerminalP2pSignal): Promise<void> {+ const shadow = this.shadowEntries.get(connId)+ if (!shadow || shadow.sessionId !== payload.sessionId || shadow.closing) return+ this.shadowEntries.delete(connId)+ this.shadowOfferSeq.delete(connId)+ const old = this.entries.get(connId)+ this.entries.set(connId, shadow)+ // notifyPeer:false — the initiator closes its own old side once OUR ack reaches it, so an abort+ // frame from here would only race that and risk landing after the initiator already moved on.+ if (old && !old.closing) void this.teardown(connId, old, 'upgraded', false)+ this.deps.sendSignal(connId, 'p2p_promote_ack', {+ sessionId: payload.sessionId,+ protocolVersion: TERMINAL_P2P_PROTOCOL_VERSION,+ })+ }}