Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

feat: implement TURN-to-direct P2P connection upgrade support with shadow session orchestration

0xken15e49df0690cc7
diff --git a/cli/src/lib/remoteRelay.spec.ts b/cli/src/lib/remoteRelay.spec.ts
index c368473be..689804e66 100644
--- a/cli/src/lib/remoteRelay.spec.ts
+++ b/cli/src/lib/remoteRelay.spec.ts
@@ -23,6 +23,13 @@ function fakeEntry(overrides: Partial<Record<string, unknown>> = {}) {
p2pRetryCount: 0,
p2pRetryLifetimeTotal: 0,
p2pRetryTimer: null,
+ upgradeAttempts: 0,
+ upgradeTimer: null,
+ upgradeShadow: null,
+ upgradeDraining: new Map<string, number>(),
+ upgradeWaitResolve: null as (() => void) | null,
+ upgradeOrphan: null,
+ upgradeDone: false,
...overrides,
}
}
@@ -284,3 +291,162 @@ describe('RemoteRelayPool live-migration of already-open streams onto p2p', () =
expect(entry.sink.sendFrame).not.toHaveBeenCalled()
})
})
+
+/** Minimal fake of the TerminalP2pInitiator surface promoteToDirect()/scheduleUpgradeAttempt() touch —
+ * a real one requires a live WebRTC negotiation, exactly why startP2p() itself has no dedicated test at
+ * this level either; this mirrors that existing limit rather than trying to route around it. */
+function fakeP2p(overrides: Partial<Record<string, unknown>> = {}) {
+ return {
+ sessionId: 'shadow-session',
+ isReady: true,
+ transport: 'direct',
+ send: vi.fn(() => true),
+ stop: vi.fn(async () => {}),
+ ...overrides,
+ }
+}
+
+describe('RemoteRelayPool TURN-to-direct upgrade', () => {
+ const pool = new RemoteRelayPool(
+ { accessToken: async () => 'unused' } as never,
+ 'ws://unused',
+ { pub: new Uint8Array(), priv: new Uint8Array() } as never,
+ { pin: () => {}, get: () => null } as never,
+ ) as unknown as {
+ scheduleUpgradeAttempt: (machineId: string, entry: ReturnType<typeof fakeEntry>) => void
+ finishUpgradeAttempt: (machineId: string, entry: ReturnType<typeof fakeEntry>) => void
+ promoteToDirect: (machineId: string, entry: ReturnType<typeof fakeEntry>, shadow: ReturnType<typeof fakeP2p>) => Promise<void>
+ demoteP2p: (machineId: string, entry: ReturnType<typeof fakeEntry>, reason: string) => void
+ // promoteToDirect() re-checks `this.entries.get(machineId) === entry` after every await (the same
+ // liveness guard scheduleP2pRetry's timer callback uses) — these tests register the fake entry here
+ // directly rather than going through a real dial(), the same shortcut fakeEntry() already takes.
+ entries: Map<string, ReturnType<typeof fakeEntry>>
+ }
+
+ it('schedules up to 3 upgrade attempts, then gives up for good', () => {
+ vi.useFakeTimers()
+ const UPGRADE_MAX = 3 // mirrors the unexported P2P_UPGRADE_MAX_ATTEMPTS constant
+ try {
+ const entry = fakeEntry()
+ for (let i = 1; i <= UPGRADE_MAX; i++) {
+ pool.scheduleUpgradeAttempt('machine-1', entry)
+ expect(entry.upgradeTimer, `attempt #${i} should have been scheduled`).not.toBeNull()
+ vi.advanceTimersByTime(60_000)
+ entry.upgradeTimer = null // the real callback nulls this before calling attemptUpgrade
+ // attemptUpgrade() itself needs a real TerminalP2pInitiator to run — simulate what it does on a
+ // failed trial (finishUpgradeAttempt is what decides whether to reschedule or give up).
+ entry.upgradeAttempts = i
+ pool.finishUpgradeAttempt('machine-1', entry)
+ }
+ expect(entry.upgradeDone).toBe(true)
+ pool.scheduleUpgradeAttempt('machine-1', entry) // refused — done for good
+ expect(entry.upgradeTimer).toBeNull()
+ } finally {
+ vi.useRealTimers()
+ }
+ })
+
+ it('does not schedule once upgradeDone, even with attempts left', () => {
+ const entry = fakeEntry({ upgradeDone: true, upgradeAttempts: 1 })
+ pool.scheduleUpgradeAttempt('machine-1', entry)
+ expect(entry.upgradeTimer).toBeNull()
+ })
+
+ it('cuts over immediately when there are no open streams to drain', async () => {
+ const old = fakeP2p({ sessionId: 'old-session' })
+ const shadow = fakeP2p({ sessionId: 'shadow-session' })
+ const entry = fakeEntry({ p2p: old, upgradeShadow: shadow })
+ pool.entries.set('machine-1', entry)
+
+ const promoted = pool.promoteToDirect('machine-1', entry, shadow)
+ // No streams means no drain wait — only the promote-ack wait is outstanding.
+ expect(entry.upgradeWaitResolve).not.toBeNull()
+ expect(entry.p2p).toBe(shadow) // cutover already happened, ack notwithstanding
+ entry.upgradeWaitResolve!()
+ await promoted
+
+ expect(old.stop).toHaveBeenCalledWith('upgraded', false)
+ expect(entry.upgradeDone).toBe(true)
+ expect(entry.upgradeOrphan).toBeNull()
+ })
+
+ it('drains open streams over the OLD connection before cutting over', async () => {
+ const old = fakeP2p({ sessionId: 'old-session' })
+ const shadow = fakeP2p({ sessionId: 'shadow-session' })
+ const entry = fakeEntry({ p2p: old, upgradeShadow: shadow, p2pStreams: new Set(['s1']) })
+ pool.entries.set('machine-1', entry)
+
+ const promoted = pool.promoteToDirect('machine-1', entry, shadow)
+ // Drain resync went out over the OLD connection — cutover has NOT happened yet.
+ expect(old.send).toHaveBeenCalledWith(JSON.stringify({ type: 'terminal_resync', payload: { streamId: 's1' } }))
+ expect(entry.p2p).toBe(old)
+ expect(entry.upgradeDraining.has('s1')).toBe(true)
+
+ entry.upgradeWaitResolve!() // simulate the drain-confirmation keyframe arriving
+ await Promise.resolve() // let promoteToDirect's await settle onto phase 2
+ await Promise.resolve()
+
+ expect(entry.p2p).toBe(shadow) // NOW cut over
+ // Phase 2's resync for 's1' rides sessionFor() again — which now routes through entry.p2p (=shadow),
+ // since p2pStreams still names 's1' and was never touched by the cutover.
+ expect(shadow.send).toHaveBeenCalledWith(JSON.stringify({ type: 'terminal_resync', payload: { streamId: 's1' } }))
+ entry.upgradeWaitResolve!() // simulate the promote ack
+ await promoted
+
+ expect(old.stop).toHaveBeenCalledWith('upgraded', false)
+ })
+
+ it('a drain timeout leaves the old connection completely untouched', async () => {
+ vi.useFakeTimers()
+ try {
+ const old = fakeP2p({ sessionId: 'old-session' })
+ const shadow = fakeP2p({ sessionId: 'shadow-session' })
+ const entry = fakeEntry({ p2p: old, upgradeShadow: shadow, p2pStreams: new Set(['s1']) })
+ pool.entries.set('machine-1', entry)
+
+ const promoted = pool.promoteToDirect('machine-1', entry, shadow)
+ await vi.advanceTimersByTimeAsync(5_000)
+ await promoted
+
+ expect(entry.p2p).toBe(old) // never cut over
+ expect(old.stop).not.toHaveBeenCalled()
+ expect(shadow.stop).toHaveBeenCalledWith('upgrade_drain_timeout', true)
+ expect(entry.upgradeShadow).toBeNull()
+ expect(entry.upgradeDraining.size).toBe(0)
+ } finally {
+ vi.useRealTimers()
+ }
+ })
+
+ it('a promote-ack timeout keeps the old connection alive as an orphan rather than guessing it is safe to close', async () => {
+ vi.useFakeTimers()
+ try {
+ const old = fakeP2p({ sessionId: 'old-session' })
+ const shadow = fakeP2p({ sessionId: 'shadow-session' })
+ const entry = fakeEntry({ p2p: old, upgradeShadow: shadow })
+ pool.entries.set('machine-1', entry)
+
+ const promoted = pool.promoteToDirect('machine-1', entry, shadow)
+ await vi.advanceTimersByTimeAsync(5_000)
+ await promoted
+
+ expect(entry.p2p).toBe(shadow) // already cut over — no reason to roll back a proven-direct pair
+ expect(old.stop).not.toHaveBeenCalled() // NOT guessed closed
+ expect(entry.upgradeOrphan).toBe(old) // left for entry-teardown cleanup instead
+ expect(entry.upgradeDone).toBe(true)
+ } finally {
+ vi.useRealTimers()
+ }
+ })
+
+ it('demoteP2p cancels an in-flight upgrade trial without touching the primary retry policy', () => {
+ const shadow = fakeP2p({ sessionId: 'shadow-session' })
+ const entry = fakeEntry({ upgradeShadow: shadow, upgradeTimer: setTimeout(() => {}, 60_000) })
+
+ pool.demoteP2p('machine-1', entry, 'send_failed')
+
+ expect(shadow.stop).toHaveBeenCalledWith('primary_demoted', true)
+ expect(entry.upgradeShadow).toBeNull()
+ expect(entry.upgradeTimer).toBeNull()
+ })
+})
diff --git a/cli/src/lib/remoteRelay.ts b/cli/src/lib/remoteRelay.ts
index 0027c8c42..652c6fba1 100644
--- a/cli/src/lib/remoteRelay.ts
+++ b/cli/src/lib/remoteRelay.ts
@@ -53,6 +53,22 @@ const P2P_RETRY_LIFETIME_CAP = 10
// on it and lets the ordinary demote-on-mismatch rule apply again.
const P2P_MIGRATION_TTL_MS = 30_000
+// A connection that reached 'direct' only via a Cloudflare TURN relay pair still costs per GB — worth
+// trying, periodically, for a truly direct pair instead. Unlike p2pRetry* above this NEVER touches the
+// live connection until a replacement has already proven itself: see scheduleUpgradeAttempt() and
+// promoteToDirect(). Capped at 3 tries and never reset (no success to reset it FOR — reaching direct
+// ends the attempts for good, and 3 relay results in a row is treated as "this path is what it is").
+const P2P_UPGRADE_MAX_ATTEMPTS = 3
+const P2P_UPGRADE_RETRY_DELAY_MS = 60_000
+// Shorter than TERMINAL_P2P_NEGOTIATION_TIMEOUT_MS's 25s: the network has already proven itself capable
+// of a full negotiation (that's how the current TURN connection exists at all), so this trial doesn't
+// need the first-contact worst-case budget.
+const P2P_UPGRADE_ATTEMPT_TIMEOUT_MS = 15_000
+// Both phases of the cutover (drain confirmation, then promote ack) wait on one round trip over an
+// already-live channel — 5s is generous for that, not for a cold negotiation.
+const P2P_UPGRADE_DRAIN_TIMEOUT_MS = 5_000
+const P2P_PROMOTE_ACK_TIMEOUT_MS = 5_000
+
export class RelayConnectError extends Error {
constructor(message: string, readonly closeCode?: number) {
super(message)
@@ -90,6 +106,26 @@ interface Entry {
* retrying forever purely because each success resets p2pRetryCount. */
p2pRetryLifetimeTotal: number
p2pRetryTimer: ReturnType<typeof setTimeout> | null
+ /** TURN-to-direct upgrade trials — see scheduleUpgradeAttempt()/attemptUpgrade()/promoteToDirect().
+ * Entirely separate bookkeeping from p2pMigrating/p2pRetry*: this never touches entry.p2p (the live,
+ * working connection) until a replacement has already proven itself direct AND drained. */
+ upgradeAttempts: number
+ upgradeTimer: ReturnType<typeof setTimeout> | null
+ upgradeShadow: TerminalP2pInitiator | null
+ /** streamId -> when its drain-barrier resync went out, RIGHT before a cutover — distinct from
+ * p2pMigrating (that one is for WS->p2p; this is for p2p(turn)->p2p(direct), and the confirmation for
+ * each arrives over a different transport, so sharing one map would let the two migrations cross-talk. */
+ upgradeDraining: Map<string, number>
+ /** Resolves whichever of the two sequential waits (drain-complete, then promote-ack) is currently
+ * outstanding — the two never overlap, so one field is enough. */
+ upgradeWaitResolve: (() => void) | null
+ /** Set only when a promote_ack never arrived: the old connection was intentionally left open rather
+ * than guessed closed (see promoteToDirect()) — cleaned up wherever the entry itself is torn down. */
+ upgradeOrphan: TerminalP2pInitiator | null
+ /** True once this entry has either cut over to a true direct pair, or spent all 3 attempts without
+ * one — either way, stop trying. Never reset (unlike p2pRetryCount): there is no "next demote" to
+ * re-arm a budget for here, since staying on direct or giving up are both permanent for this entry. */
+ upgradeDone: boolean
}
export interface RelaySession {
@@ -148,6 +184,9 @@ export class RemoteRelayPool {
if (entry.heartbeatTimer) clearInterval(entry.heartbeatTimer)
if (entry.lingerTimer) clearTimeout(entry.lingerTimer)
if (entry.p2pRetryTimer) clearTimeout(entry.p2pRetryTimer)
+ if (entry.upgradeTimer) clearTimeout(entry.upgradeTimer)
+ void entry.upgradeShadow?.stop('invalidated', false)
+ void entry.upgradeOrphan?.stop('invalidated', false)
// The caller is invalidating so it can immediately acquire() a fresh entry on the SAME local
// connection (it just got a forceReconnect select) — null this out first so the generic
// `ws.on('close', ...)` cleanup below doesn't turn around and close that same local socket via a
@@ -232,6 +271,13 @@ export class RemoteRelayPool {
p2pRetryCount: 0,
p2pRetryLifetimeTotal: 0,
p2pRetryTimer: null,
+ upgradeAttempts: 0,
+ upgradeTimer: null,
+ upgradeShadow: null,
+ upgradeDraining: new Map(),
+ upgradeWaitResolve: null,
+ upgradeOrphan: null,
+ upgradeDone: false,
}
// Two phases before this connection is usable: (1) machine_select ack, (2) this daemon's own
// e2e_hello/e2e_welcome as the "client" role — see lib/e2ee/relayClient.ts. Only once BOTH are done
@@ -357,7 +403,24 @@ export class RemoteRelayPool {
if (!plain) return
const type = typeof plain.type === 'string' ? plain.type : ''
if (TERMINAL_P2P_SIGNAL_TYPES.has(type)) {
- void entry.p2p?.handleSignal(type, plain.payload)
+ // p2p_promote/p2p_promote_ack belong to the upgrade orchestration below, not to
+ // TerminalP2pInitiator's own protocol — it would just silently no-op on them.
+ if (type === 'p2p_promote_ack') {
+ const ackPayload = plain.payload as { sessionId?: unknown } | undefined
+ if (typeof ackPayload?.sessionId === 'string' && ackPayload.sessionId === entry.p2p?.sessionId) {
+ entry.upgradeWaitResolve?.()
+ }
+ return
+ }
+ // A shadow trial negotiates its OWN session alongside entry.p2p (the live primary) — route by
+ // sessionId so its answer/candidates/abort reach it instead of being checked against (and
+ // silently dropped by) the primary's handleSignal, which only matches its own sessionId.
+ const signalSessionId = (plain.payload as { sessionId?: unknown } | undefined)?.sessionId
+ if (entry.upgradeShadow && typeof signalSessionId === 'string' && signalSessionId === entry.upgradeShadow.sessionId) {
+ void entry.upgradeShadow.handleSignal(type, plain.payload)
+ } else {
+ void entry.p2p?.handleSignal(type, plain.payload)
+ }
return
}
// Forward the real frame FIRST: for `terminal_ready`, the Desktop app's TerminalSession learns
@@ -382,6 +445,9 @@ export class RemoteRelayPool {
ws.on('close', (code, reasonBuf) => {
if (entry.heartbeatTimer) clearInterval(entry.heartbeatTimer)
if (entry.p2pRetryTimer) clearTimeout(entry.p2pRetryTimer)
+ if (entry.upgradeTimer) clearTimeout(entry.upgradeTimer)
+ void entry.upgradeShadow?.stop('relay_closed', false)
+ void entry.upgradeOrphan?.stop('relay_closed', false)
void entry.p2p?.stop('relay_closed', false)
entry.p2p = null
this.entries.delete(machineId)
@@ -449,6 +515,9 @@ export class RemoteRelayPool {
entry.lingerTimer = setTimeout(() => {
if (!entry.sink) {
if (entry.p2pRetryTimer) clearTimeout(entry.p2pRetryTimer)
+ if (entry.upgradeTimer) clearTimeout(entry.upgradeTimer)
+ void entry.upgradeShadow?.stop('idle', false)
+ void entry.upgradeOrphan?.stop('idle', false)
void entry.p2p?.stop('idle', false)
entry.p2p = null
try { entry.ws.close(1000, 'idle') } catch { /* ignore */ }
@@ -488,6 +557,7 @@ export class RemoteRelayPool {
console.log(`[p2p] connected · machine=${sid(machineId)} via=${relayed ? 'turn' : 'direct'} setup=${Math.round(setupMs)}ms`)
this.reportP2pResult(entry, 'direct', setupMs, relayed ? 'relayed' : undefined)
this.promoteOpenStreams(machineId, entry)
+ if (relayed && !entry.upgradeDone) this.scheduleUpgradeAttempt(machineId, entry)
} else if (state === 'failed' && !wasDirect) {
console.log(`[p2p] gave up · machine=${sid(machineId)} reason=${reason ?? 'unknown'} after=${Math.round(setupMs)}ms`
+ ` · ${p2p.negotiationDetail} — terminals stay on the ws relay`)
@@ -515,6 +585,12 @@ export class RemoteRelayPool {
// stream's bytes are now genuinely arriving over p2p — safe to re-arm the ordinary
// demote-on-mismatch rule for it (see the guard in the ws binary handler above).
entry.p2pMigrating.delete(clear.streamId)
+ // Drain-barrier confirmation for a TURN-to-direct upgrade in flight (promoteToDirect() phase 1):
+ // this keyframe is the OLD (still-primary) connection's answer to the resync it sent to itself
+ // over p2p, proving everything before that point has been accounted for.
+ if (entry.upgradeDraining.delete(clear.streamId) && entry.upgradeDraining.size === 0) {
+ entry.upgradeWaitResolve?.()
+ }
const local = encodeTerminalLocal(clear)
if (local) entry.sink?.sendBinary(local)
return
@@ -579,6 +655,12 @@ export class RemoteRelayPool {
}
private demoteP2p(machineId: string, entry: Entry, reason: string): void {
+ // The primary just died (or is dying) — an upgrade trial in flight for it no longer means anything,
+ // and scheduleP2pRetry() below is about to take over recovery via the normal path anyway.
+ if (entry.upgradeTimer) { clearTimeout(entry.upgradeTimer); entry.upgradeTimer = null }
+ if (entry.upgradeShadow) { void entry.upgradeShadow.stop('primary_demoted', true); entry.upgradeShadow = null }
+ entry.upgradeDraining.clear()
+ entry.upgradeWaitResolve = null
const p2p = entry.p2p
entry.p2p = null
const streamIds = [...entry.p2pStreams]
@@ -656,6 +738,172 @@ export class RemoteRelayPool {
// responder committed its own flip. Until then the demote-on-mismatch rule stays suppressed for it.
}
+ /**
+ * Kicks off a periodic attempt to replace a TURN-relayed p2p connection with a true direct one.
+ * Never touches entry.p2p until attemptUpgrade()'s shadow has already proven itself — see the Entry
+ * interface's doc on upgradeShadow/upgradeDraining.
+ */
+ private scheduleUpgradeAttempt(machineId: string, entry: Entry): void {
+ if (entry.upgradeTimer || entry.upgradeShadow || entry.upgradeDone) return
+ if (entry.upgradeAttempts >= P2P_UPGRADE_MAX_ATTEMPTS) { entry.upgradeDone = true; return }
+ entry.upgradeTimer = setTimeout(() => {
+ entry.upgradeTimer = null
+ if (this.entries.get(machineId) !== entry) return // entry was torn down/replaced meanwhile
+ this.attemptUpgrade(machineId, entry)
+ }, P2P_UPGRADE_RETRY_DELAY_MS)
+ entry.upgradeTimer.unref?.()
+ }
+
+ /** One trial: negotiate a brand-new (shadow) p2p connection from scratch, entirely independent of the
+ * live primary, and see whether IT lands on a direct pair. The primary keeps serving traffic exactly
+ * as before for the whole trial — nothing here is wired into entry.p2pStreams/sessionFor() routing
+ * until promoteToDirect() has already proven success, so nothing here can disrupt it. */
+ private attemptUpgrade(machineId: string, entry: Entry): void {
+ if (entry.upgradeDone || !entry.p2p?.isReady || entry.p2p.transport !== 'relay' || !entry.p2pPolicy) return
+ entry.upgradeAttempts++
+ const policy = entry.p2pPolicy
+ const shadow: TerminalP2pInitiator = new TerminalP2pInitiator({
+ policy,
+ upgrade: true,
+ sendSignal: (type, payload) => {
+ const wrapped = entry.crypto.wrapOutgoing({ type, payload })
+ try { entry.ws.send(JSON.stringify(wrapped)) } catch { /* relay close handles cleanup */ }
+ },
+ // Identical to the primary's own wiring — correct both before promotion (nothing routes real data
+ // here yet, so this never actually fires) and after (once promoted, this IS the primary).
+ onData: (data) => this.handleP2pData(entry, data),
+ onStep: (step, elapsedMs) => {
+ console.log(`[p2p-upgrade] step · machine=${sid(machineId)} attempt=${entry.upgradeAttempts} ${step} +${Math.round(elapsedMs)}ms`)
+ },
+ onState: (state, setupMs, reason) => {
+ console.log(`[p2p-upgrade] ${state} · machine=${sid(machineId)} attempt=${entry.upgradeAttempts}`
+ + ` +${Math.round(setupMs)}ms${reason ? ` reason=${reason}` : ''}`)
+ },
+ onUnavailable: (reason) => {
+ if (entry.p2p === shadow) {
+ // Already promoted and now failing for real — an ordinary primary failure from here on.
+ this.demoteP2p(machineId, entry, reason)
+ } else if (entry.upgradeShadow === shadow) {
+ // Died as a trial, before cutover — the primary was never touched. Just abandon this attempt.
+ entry.upgradeShadow = null
+ entry.upgradeDraining.clear()
+ entry.upgradeWaitResolve = null
+ this.finishUpgradeAttempt(machineId, entry)
+ }
+ },
+ })
+ entry.upgradeShadow = shadow
+ shadow.start()
+ void this.runUpgradeAttempt(machineId, entry, shadow)
+ }
+
+ private async runUpgradeAttempt(machineId: string, entry: Entry, shadow: TerminalP2pInitiator): Promise<void> {
+ const ok = await shadow.waitUntilReady(P2P_UPGRADE_ATTEMPT_TIMEOUT_MS)
+ if (entry.upgradeShadow !== shadow || this.entries.get(machineId) !== entry) return // superseded/torn down meanwhile
+ if (!ok || shadow.transport === 'relay') {
+ entry.upgradeShadow = null
+ void shadow.stop('upgrade_no_gain', true)
+ this.finishUpgradeAttempt(machineId, entry)
+ return
+ }
+ await this.promoteToDirect(machineId, entry, shadow)
+ }
+
+ private finishUpgradeAttempt(machineId: string, entry: Entry): void {
+ if (entry.upgradeAttempts >= P2P_UPGRADE_MAX_ATTEMPTS) {
+ entry.upgradeDone = true
+ console.log(`[p2p-upgrade] giving up · machine=${sid(machineId)} staying on turn after ${entry.upgradeAttempts} attempts`)
+ return
+ }
+ this.scheduleUpgradeAttempt(machineId, entry)
+ }
+
+ /**
+ * Cuts entry.p2p over to `shadow`, which has already proven itself a true direct pair. Every bail-out
+ * before the actual swap (the `entry.p2p = shadow` line) leaves the primary completely untouched —
+ * that line is the one moment this can no longer be undone, and everything before it is written to
+ * make that moment as late and as certain as possible.
+ */
+ private async promoteToDirect(machineId: string, entry: Entry, shadow: TerminalP2pInitiator): Promise<void> {
+ const old = entry.p2p
+ if (!old || entry.upgradeShadow !== shadow || this.entries.get(machineId) !== entry) {
+ void shadow.stop('upgrade_stale', true)
+ if (entry.upgradeShadow === shadow) entry.upgradeShadow = null
+ this.finishUpgradeAttempt(machineId, entry)
+ return
+ }
+ const streamIds = [...entry.p2pStreams]
+ if (streamIds.length > 0) {
+ // Phase 1 — drain barrier over the OLD (still-primary, still fully live) connection.
+ const drained = await this.waitForUpgradeMilestone(entry, () => {
+ for (const streamId of streamIds) entry.upgradeDraining.set(streamId, Date.now())
+ for (const streamId of streamIds) {
+ void this.sessionFor(machineId, entry).send({ type: 'terminal_resync', payload: { streamId } })
+ }
+ }, P2P_UPGRADE_DRAIN_TIMEOUT_MS)
+ if (entry.upgradeShadow !== shadow || this.entries.get(machineId) !== entry) {
+ // Torn down/superseded while draining — old was never touched either way, just clean up.
+ entry.upgradeDraining.clear()
+ void shadow.stop('upgrade_stale', true)
+ return
+ }
+ if (!drained) {
+ // Timed out — OLD IS STILL COMPLETELY UNTOUCHED. Abandon just this attempt.
+ entry.upgradeDraining.clear()
+ entry.upgradeShadow = null
+ void shadow.stop('upgrade_drain_timeout', true)
+ this.finishUpgradeAttempt(machineId, entry)
+ return
+ }
+ }
+ // Phase 2 — the actual cutover. entry.p2pStreams is untouched: every streamId in it was already
+ // p2p and stays p2p, only the object behind entry.p2p changes.
+ entry.p2p = shadow
+ entry.upgradeShadow = null
+ const acked = await this.waitForUpgradeMilestone(entry, () => {
+ for (const streamId of streamIds) {
+ void this.sessionFor(machineId, entry).send({ type: 'terminal_resync', payload: { streamId } })
+ }
+ const wrapped = entry.crypto.wrapOutgoing({
+ type: 'p2p_promote',
+ payload: { sessionId: shadow.sessionId, protocolVersion: TERMINAL_P2P_PROTOCOL_VERSION },
+ })
+ try { entry.ws.send(JSON.stringify(wrapped)) } catch { /* relay close handles cleanup */ }
+ }, P2P_PROMOTE_ACK_TIMEOUT_MS)
+ entry.upgradeDone = true // either way, this entry is done trying — see the field's own doc comment
+ if (acked) {
+ void old.stop('upgraded', false)
+ console.log(`[p2p-upgrade] promoted · machine=${sid(machineId)} attempt=${entry.upgradeAttempts}`)
+ } else {
+ // Ack never arrived — keep OLD alive rather than guess it is safe to close; entry.upgradeOrphan is
+ // closed wherever the entry itself is torn down (invalidate/ws-close/idle-linger).
+ entry.upgradeOrphan = old
+ console.log(`[p2p-upgrade] promote ack timeout · machine=${sid(machineId)} — keeping the old connection`
+ + ' open until this entry is torn down')
+ }
+ }
+
+ /** One shared waiter shape for promoteToDirect()'s two sequential phases (drain-complete, then
+ * promote-ack) — they never overlap, so entry.upgradeWaitResolve is reused rather than doubled.
+ * `start` runs AFTER the timeout/resolver are armed, so a same-tick resolve inside it can never race
+ * past a listener that is not wired up yet. */
+ private waitForUpgradeMilestone(entry: Entry, start: () => void, timeoutMs: number): Promise<boolean> {
+ return new Promise<boolean>((resolve) => {
+ let settled = false
+ const done = (ok: boolean): void => {
+ if (settled) return
+ settled = true
+ clearTimeout(timer)
+ entry.upgradeWaitResolve = null
+ resolve(ok)
+ }
+ const timer = setTimeout(() => done(false), timeoutMs)
+ timer.unref?.()
+ entry.upgradeWaitResolve = () => done(true)
+ start()
+ })
+ }
+
private reportP2pResult(entry: Entry, outcome: string, setupMs?: number, reason?: string): void {
try {
entry.ws.send(JSON.stringify({
diff --git a/cli/src/lib/terminalP2p.ts b/cli/src/lib/terminalP2p.ts
index 071b1e0bd..aa527a46d 100644
--- a/cli/src/lib/terminalP2p.ts
+++ b/cli/src/lib/terminalP2p.ts
@@ -27,6 +27,12 @@ export const TERMINAL_P2P_SIGNAL_TYPES = new Set([
'p2p_answer',
'p2p_ice_candidate',
'p2p_abort',
+ // Owned entirely by RemoteRelayPool's TURN-to-direct upgrade orchestration (remoteRelay.ts), not by
+ // TerminalP2pInitiator/TerminalP2pResponderPool's own protocol — these two just need to ride the same
+ // encrypted signaling channel. p2p_promote: "cut the primary over to this already-negotiated shadow
+ // session now." p2p_promote_ack: the responder confirming it did.
+ 'p2p_promote',
+ 'p2p_promote_ack',
])
export const TERMINAL_P2P_DOWN_TYPES = new Set([
@@ -70,6 +76,9 @@ export interface TerminalP2pSignal {
* two peers, the credential is short-lived, and the backend minted it in the first place.
*/
turn?: TerminalP2pTurn
+ /** Set on the offer only when this negotiation is a TURN-to-direct upgrade trial for an already-live
+ * connId, not a fresh session — see TerminalP2pResponderPool.acceptUpgradeOffer(). */
+ upgrade?: true
}
export interface TerminalP2pInitiatorDeps {
@@ -82,6 +91,10 @@ export interface TerminalP2pInitiatorDeps {
selectStunUrls?: StunSelector
/** Coarse negotiation milestones, for working out WHERE a slow setup spends its time. */
onStep?: (step: string, elapsedMs: number) => void
+ /** This instance is a shadow trial negotiated alongside an already-live connection (see
+ * RemoteRelayPool's TURN-to-direct upgrade), not a fresh session — tags the offer so the responder
+ * routes it to acceptUpgradeOffer() instead of tearing down the connId's live entry. */
+ upgrade?: boolean
}
type ReadyWaiter = (ready: boolean) => void
@@ -273,6 +286,7 @@ export class TerminalP2pInitiator {
// need not agree on a server — a srflx candidate is each peer's own public address.
stunUrls: this.deps.policy.stunUrls,
...(this.deps.policy.turn ? { turn: this.deps.policy.turn } : {}),
+ ...(this.deps.upgrade ? { upgrade: true as const } : {}),
})
this.step('offer-sent')
} catch {
@@ -418,6 +432,13 @@ export class TerminalP2pResponderPool {
* loser's peer connection would be orphaned by the winner's entries.set(). Bump on entry, re-check
* after every await. Pre-existing race — the STUN selection below only widens the window. */
private readonly offerSeq = new Map<string, number>()
+ /** One live connId's primary connection may have an upgrade trial negotiating alongside it — kept in
+ * a separate map, keyed by the SAME connId, so the primary is never touched while a trial is live.
+ * See acceptUpgradeOffer()/promote(). */
+ private readonly shadowEntries = new Map<string, ResponderEntry>()
+ /** Same purpose as offerSeq, but tracked separately: a primary offer and an upgrade offer for the
+ * same connId are unrelated negotiations and must not invalidate each other's generation. */
+ private readonly shadowOfferSeq = new Map<string, number>()
private readonly selectStunUrls: StunSelector
constructor(private readonly deps: TerminalP2pResponderPoolDeps) {
@@ -429,16 +450,33 @@ export class TerminalP2pResponderPool {
const payload = parseSignal(value)
if (!payload) return true
if (type === 'p2p_offer' && typeof payload.sdp === 'string') {
- await this.acceptOffer(connId, payload)
+ if (payload.upgrade === true) await this.acceptUpgradeOffer(connId, payload)
+ else await this.acceptOffer(connId, payload)
+ return true
+ }
+ if (type === 'p2p_promote') {
+ await this.promote(connId, payload)
return true
}
const entry = this.entries.get(connId)
- if (!entry || entry.sessionId !== payload.sessionId) return true
- try {
- if (type === 'p2p_ice_candidate') await entry.pc.addIceCandidate(payload.candidate ?? null)
- else if (type === 'p2p_abort') await this.closeConnection(connId, payload.reason || 'peer_aborted', false)
- } catch {
- await this.closeConnection(connId, 'signal_invalid')
+ if (entry && entry.sessionId === payload.sessionId) {
+ try {
+ if (type === 'p2p_ice_candidate') await entry.pc.addIceCandidate(payload.candidate ?? null)
+ else if (type === 'p2p_abort') await this.closeConnection(connId, payload.reason || 'peer_aborted', false)
+ } catch {
+ await this.closeConnection(connId, 'signal_invalid')
+ }
+ return true
+ }
+ // Not the primary's session — try the upgrade trial, if one is in flight for this connId.
+ const shadow = this.shadowEntries.get(connId)
+ if (shadow && shadow.sessionId === payload.sessionId) {
+ try {
+ if (type === 'p2p_ice_candidate') await shadow.pc.addIceCandidate(payload.candidate ?? null)
+ else if (type === 'p2p_abort') await this.closeShadow(connId, payload.reason || 'peer_aborted')
+ } catch {
+ await this.closeShadow(connId, 'signal_invalid')
+ }
}
return true
}
@@ -456,16 +494,13 @@ export class TerminalP2pResponderPool {
}
}
- async closeConnection(connId: string, reason = 'closed', notifyPeer = true): Promise<void> {
- const entry = this.entries.get(connId)
- if (!entry || entry.closing) return
+ /** Shared teardown body for both the primary map and the shadow map — takes the entry directly
+ * rather than looking it up by connId, because by the time promote() needs to close the OLD primary,
+ * `entries.get(connId)` already points at the just-promoted shadow. Looking it up again here would
+ * close the wrong connection. */
+ private async teardown(connId: string, entry: ResponderEntry, reason: string, notifyPeer: boolean): Promise<void> {
+ if (entry.closing) return
entry.closing = true
- this.entries.delete(connId)
- // Every reason but 'superseded' retires the connId for good, so drop its generation counter too or
- // the map grows for the life of the daemon. 'superseded' is excluded because that call comes from
- // acceptOffer itself, which has already claimed the current generation and still needs it as its
- // own liveness check across the awaits that follow.
- if (reason !== 'superseded') this.offerSeq.delete(connId)
clearTimeout(entry.timeout)
if (notifyPeer) {
this.deps.sendSignal(connId, 'p2p_abort', {
@@ -478,11 +513,40 @@ export class TerminalP2pResponderPool {
await entry.pc.close().catch(() => { /* best effort */ })
}
+ async closeConnection(connId: string, reason = 'closed', notifyPeer = true): Promise<void> {
+ const entry = this.entries.get(connId)
+ if (entry && !entry.closing) {
+ this.entries.delete(connId)
+ // Every reason but 'superseded' retires the connId for good, so drop its generation counter too
+ // or the map grows for the life of the daemon. 'superseded' is excluded because that call comes
+ // from acceptOffer itself, which has already claimed the current generation and still needs it as
+ // its own liveness check across the awaits that follow.
+ if (reason !== 'superseded') this.offerSeq.delete(connId)
+ await this.teardown(connId, entry, reason, notifyPeer)
+ }
+ // A trial upgrade tied to this connId's primary no longer means anything once the primary itself is
+ // gone (or is being superseded by a fresh, non-upgrade offer) — retire it too rather than leaking it.
+ await this.closeShadow(connId, reason)
+ }
+
+ private async closeShadow(connId: string, reason: string): Promise<void> {
+ const shadow = this.shadowEntries.get(connId)
+ if (!shadow || shadow.closing) return
+ this.shadowEntries.delete(connId)
+ this.shadowOfferSeq.delete(connId)
+ // The initiator tracks its own shadow's lifecycle locally (waitUntilReady's own timeout) rather than
+ // depending on a signal from here, so this never notifies the peer — one fewer round trip for a
+ // trial that, by definition, has no traffic riding on it yet.
+ await this.teardown(connId, shadow, reason, false)
+ }
+
async stop(): Promise<void> {
await Promise.all([...this.entries.keys()].map((connId) => this.closeConnection(connId, 'shutdown', false)))
+ await Promise.all([...this.shadowEntries.keys()].map((connId) => this.closeShadow(connId, 'shutdown')))
// Also drops generations for offers still inside their STUN race, which makes them bail instead of
// building a peer connection nothing would ever close.
this.offerSeq.clear()
+ this.shadowOfferSeq.clear()
}
private async acceptOffer(connId: string, payload: TerminalP2pSignal): Promise<void> {
@@ -490,13 +554,42 @@ export class TerminalP2pResponderPool {
this.offerSeq.set(connId, seq)
await this.closeConnection(connId, 'superseded', false)
if (this.offerSeq.get(connId) !== seq) return
+ const entry = await this.buildResponderEntry(connId, payload, () => this.offerSeq.get(connId) === seq)
+ if (!entry) return
+ this.entries.set(connId, entry)
+ this.wireResponderEntry(connId, entry, () => this.entries.get(connId) === entry, (reason) => this.closeConnection(connId, reason))
+ await this.answerOffer(connId, entry, payload, () => this.entries.get(connId) === entry, (reason) => this.closeConnection(connId, reason))
+ }
+
+ /** Mirrors acceptOffer(), but never touches the connId's live primary entry — see the class-level
+ * doc on shadowEntries. Failure at any point here costs nothing but this one trial. */
+ private async acceptUpgradeOffer(connId: string, payload: TerminalP2pSignal): Promise<void> {
+ const seq = (this.shadowOfferSeq.get(connId) ?? 0) + 1
+ this.shadowOfferSeq.set(connId, seq)
+ await this.closeShadow(connId, 'superseded') // retire a stale trial for this connId, if any
+ if (this.shadowOfferSeq.get(connId) !== seq) return
+ const entry = await this.buildResponderEntry(connId, payload, () => this.shadowOfferSeq.get(connId) === seq)
+ if (!entry) return
+ this.shadowEntries.set(connId, entry)
+ this.wireResponderEntry(connId, entry, () => this.shadowEntries.get(connId) === entry, (reason) => this.closeShadow(connId, reason))
+ await this.answerOffer(connId, entry, payload, () => this.shadowEntries.get(connId) === entry, (reason) => this.closeShadow(connId, reason))
+ }
+
+ /** STUN race + RTCPeerConnection construction shared by acceptOffer/acceptUpgradeOffer. `stillCurrent`
+ * is re-checked after the only await in here (the STUN race) so a superseded offer never publishes a
+ * peer connection nothing would ever close. Returns null when superseded — caller just returns. */
+ private async buildResponderEntry(
+ connId: string,
+ payload: TerminalP2pSignal,
+ stillCurrent: () => boolean,
+ ): Promise<ResponderEntry | null> {
const offeredStunUrls = Array.isArray(payload.stunUrls)
? payload.stunUrls.filter((url): url is string => typeof url === 'string' && /^stuns?:/i.test(url)).slice(0, 8)
: []
// The offerer raced these too, and may well have landed on a different server. That is fine: a
// srflx candidate is each peer's own public address, so the two sides need not agree on who to ask.
const selection = await this.selectStunUrls(offeredStunUrls)
- if (this.offerSeq.get(connId) !== seq) return
+ if (!stillCurrent()) return null
const turn = turnChoice(readTurn(payload.turn), selection.udpReachable)
const pc = new RTCPeerConnection(peerConfig(selection.urls, turn))
const entry: ResponderEntry = {
@@ -506,51 +599,80 @@ export class TerminalP2pResponderPool {
ready: false,
closing: false,
timeout: setTimeout(() => {
- this.deps.onUnavailable?.(connId, 'negotiation_timeout')
- void this.closeConnection(connId, 'negotiation_timeout')
+ if (entry.ready) this.deps.onUnavailable?.(connId, 'negotiation_timeout')
+ void (stillCurrent() ? this.closeConnection(connId, 'negotiation_timeout') : this.closeShadow(connId, 'negotiation_timeout'))
}, TERMINAL_P2P_NEGOTIATION_TIMEOUT_MS),
}
entry.timeout.unref?.()
- this.entries.set(connId, entry)
- pc.connectionStateChange.subscribe((state) => {
- if (this.entries.get(connId) !== entry || entry.closing) return
+ return entry
+ }
+
+ /** connectionStateChange/onDataChannel wiring shared by acceptOffer/acceptUpgradeOffer. `notifyPeer`
+ * reporting via deps.onUnavailable is deliberately gated on `isPrimary()` — a trial failing before it
+ * is ever promoted is not the connId's p2p becoming unavailable, since the real primary (if any) is
+ * untouched throughout. */
+ private wireResponderEntry(
+ connId: string,
+ entry: ResponderEntry,
+ isCurrent: () => boolean,
+ close: (reason: string) => Promise<void>,
+ ): void {
+ entry.pc.connectionStateChange.subscribe((state) => {
+ if (!isCurrent() || entry.closing) return
if (state === 'failed' || state === 'disconnected') {
- if (entry.ready) this.deps.onUnavailable?.(connId, `peer_${state}`)
- void this.closeConnection(connId, `peer_${state}`)
+ if (entry.ready && this.entries.get(connId) === entry) this.deps.onUnavailable?.(connId, `peer_${state}`)
+ void close(`peer_${state}`)
}
})
- pc.onDataChannel.subscribe((channel) => {
- if (this.entries.get(connId) !== entry || channel.label !== TERMINAL_P2P_CHANNEL) {
+ entry.pc.onDataChannel.subscribe((channel) => {
+ if (!isCurrent() || channel.label !== TERMINAL_P2P_CHANNEL) {
channel.close()
return
}
entry.channel = channel
channel.bufferedAmountLowThreshold = 256 * 1024
channel.stateChanged.subscribe((state) => {
- if (this.entries.get(connId) !== entry || entry.closing) return
+ if (!isCurrent() || entry.closing) return
if (state === 'open') {
entry.ready = true
clearTimeout(entry.timeout)
} else if ((state === 'closed' || state === 'closing') && entry.ready) {
entry.ready = false
- this.deps.onUnavailable?.(connId, 'channel_closed')
- void this.closeConnection(connId, 'channel_closed')
+ if (this.entries.get(connId) === entry) this.deps.onUnavailable?.(connId, 'channel_closed')
+ void close('channel_closed')
}
})
channel.onMessage.subscribe((data) => {
- if (this.entries.get(connId) === entry && !entry.closing) this.deps.onData(connId, data)
+ // True for the trial's own channel too — never actually reached before promote() moves this
+ // entry into `entries`, since the initiator only ever writes to its OWN entry.p2p, which stays
+ // the primary throughout a trial. Kept identical to the pre-refactor primary-only check instead
+ // of narrowing it, so promotion needs no re-wiring here.
+ if ((this.entries.get(connId) === entry || this.shadowEntries.get(connId) === entry) && !entry.closing) {
+ this.deps.onData(connId, data)
+ }
})
channel.error.subscribe(() => {
- if (entry.ready) this.deps.onUnavailable?.(connId, 'channel_error')
- void this.closeConnection(connId, 'channel_error')
+ if (entry.ready && this.entries.get(connId) === entry) this.deps.onUnavailable?.(connId, 'channel_error')
+ void close('channel_error')
})
})
+ }
+
+ /** setRemoteDescription/createAnswer/setLocalDescription + sending the answer — shared tail of
+ * acceptOffer/acceptUpgradeOffer. */
+ private async answerOffer(
+ connId: string,
+ entry: ResponderEntry,
+ payload: TerminalP2pSignal,
+ isCurrent: () => boolean,
+ close: (reason: string) => Promise<void>,
+ ): Promise<void> {
try {
- await pc.setRemoteDescription({ type: 'offer', sdp: payload.sdp! })
- const answer = await pc.createAnswer()
- await pc.setLocalDescription(answer)
- if (this.entries.get(connId) !== entry || entry.closing) return
- const local = pc.localDescription
+ await entry.pc.setRemoteDescription({ type: 'offer', sdp: payload.sdp! })
+ const answer = await entry.pc.createAnswer()
+ await entry.pc.setLocalDescription(answer)
+ if (!isCurrent() || entry.closing) return
+ const local = entry.pc.localDescription
if (!local) throw new Error('local_description_missing')
this.deps.sendSignal(connId, 'p2p_answer', {
sessionId: payload.sessionId,
@@ -558,7 +680,26 @@ export class TerminalP2pResponderPool {
sdp: local.sdp,
})
} catch {
- await this.closeConnection(connId, 'answer_failed')
+ await close('answer_failed')
}
}
+
+ /** Cuts the connId's primary connection over to an already-negotiated-and-ready shadow trial. The
+ * shadow must already be at the sessionId named in payload — this never builds a peer connection
+ * itself, only swaps which one `entries` points at and retires whichever was there before. */
+ private async promote(connId: string, payload: TerminalP2pSignal): Promise<void> {
+ const shadow = this.shadowEntries.get(connId)
+ if (!shadow || shadow.sessionId !== payload.sessionId || shadow.closing) return
+ this.shadowEntries.delete(connId)
+ this.shadowOfferSeq.delete(connId)
+ const old = this.entries.get(connId)
+ this.entries.set(connId, shadow)
+ // notifyPeer:false — the initiator closes its own old side once OUR ack reaches it, so an abort
+ // frame from here would only race that and risk landing after the initiator already moved on.
+ if (old && !old.closing) void this.teardown(connId, old, 'upgraded', false)
+ this.deps.sendSignal(connId, 'p2p_promote_ack', {
+ sessionId: payload.sessionId,
+ protocolVersion: TERMINAL_P2P_PROTOCOL_VERSION,
+ })
+ }
}