Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

feat: implement P2P retry policy and live-migration for remote terminal streams

0xken159ba777d6229e
diff --git a/cli/src/backendSocket.ts b/cli/src/backendSocket.ts
index 8606bb8d3..9b887fa4c 100644
--- a/cli/src/backendSocket.ts
+++ b/cli/src/backendSocket.ts
@@ -820,6 +820,18 @@ export class BackendSocket {
return
}
const streamId = typeof payload.streamId === 'string' ? payload.streamId : ''
+ // Live-migration promotion: the client's remoteRelay.ts sends a SECOND terminal_resync over p2p
+ // (after the first one, over relay, already drained/snapshotted the stream) once its own p2p
+ // channel is ready — arriving here is our signal to start routing this stream's OUTPUT over p2p
+ // too, mirroring what the client just did on its side. hasStream() guards against promoting a
+ // streamId whose pane was closed in the same instant the migration was in flight.
+ if (type === 'terminal_resync' && transport === 'p2p' && streamId
+ && this.terminalStreams?.hasStream(connId, streamId)) {
+ let streams = this.p2pStreams.get(connId)
+ if (!streams) { streams = new Set(); this.p2pStreams.set(connId, streams) }
+ streams.add(streamId)
+ return
+ }
if (transport === 'relay' && streamId) this.p2pStreams.get(connId)?.delete(streamId)
}
diff --git a/cli/src/lib/remoteRelay.spec.ts b/cli/src/lib/remoteRelay.spec.ts
index 696578259..c368473be 100644
--- a/cli/src/lib/remoteRelay.spec.ts
+++ b/cli/src/lib/remoteRelay.spec.ts
@@ -18,6 +18,11 @@ function fakeEntry(overrides: Partial<Record<string, unknown>> = {}) {
p2pPolicy: null,
p2pPendingOpens: new Set<string>(),
p2pStreams: new Set<string>(),
+ streams: new Set<string>(),
+ p2pMigrating: new Map<string, number>(),
+ p2pRetryCount: 0,
+ p2pRetryLifetimeTotal: 0,
+ p2pRetryTimer: null,
...overrides,
}
}
@@ -31,7 +36,7 @@ describe('RemoteRelayPool reports terminal_link_mode to the local sink', () => {
{ pin: () => {}, get: () => null } as never,
) as unknown as {
noteTerminalResponse: (entry: ReturnType<typeof fakeEntry>, frame: Record<string, unknown>, transport: 'p2p' | 'relay') => void
- demoteP2p: (entry: ReturnType<typeof fakeEntry>, reason: string) => void
+ demoteP2p: (machineId: string, entry: ReturnType<typeof fakeEntry>, reason: string) => void
}
it('a terminal_ready confirmed on p2p reports mode:p2p, matching p2pStreams', () => {
@@ -117,7 +122,7 @@ describe('RemoteRelayPool reports terminal_link_mode to the local sink', () => {
it('demoteP2p reports mode:relay for every stream it drops', () => {
const entry = fakeEntry({ p2pStreams: new Set(['stream-a', 'stream-b']) })
- pool.demoteP2p(entry, 'send_failed')
+ pool.demoteP2p('machine-1', entry, 'send_failed')
expect(entry.p2pStreams.size).toBe(0)
expect(entry.sink.sendFrame).toHaveBeenCalledWith({
@@ -132,8 +137,150 @@ describe('RemoteRelayPool reports terminal_link_mode to the local sink', () => {
it('demoteP2p on a connection with no active p2p streams reports nothing', () => {
const entry = fakeEntry()
- pool.demoteP2p(entry, 'send_failed')
+ pool.demoteP2p('machine-1', entry, 'send_failed')
expect(entry.sink.sendFrame).not.toHaveBeenCalled()
})
})
+
+describe('RemoteRelayPool p2p retry policy', () => {
+ const pool = new RemoteRelayPool(
+ { accessToken: async () => 'unused' } as never,
+ 'ws://unused',
+ { pub: new Uint8Array(), priv: new Uint8Array() } as never,
+ { pin: () => {}, get: () => null } as never,
+ ) as unknown as {
+ scheduleP2pRetry: (machineId: string, entry: ReturnType<typeof fakeEntry>) => void
+ demoteP2p: (machineId: string, entry: ReturnType<typeof fakeEntry>, reason: string) => void
+ }
+
+ it('schedules up to 3 retries (i.e. 4 attempts total: 1 initial + 3 retries), then refuses a 4th', () => {
+ vi.useFakeTimers()
+ const RETRY_MAX = 3 // mirrors the unexported P2P_RETRY_MAX constant in remoteRelay.ts
+ try {
+ const entry = fakeEntry()
+ // The initial attempt itself is NOT scheduled through scheduleP2pRetry (it fires unconditionally
+ // at e2e_welcome) — this exercises only the retries that follow each subsequent failure.
+ for (let i = 1; i <= RETRY_MAX; i++) {
+ pool.scheduleP2pRetry('machine-1', entry)
+ expect(entry.p2pRetryTimer, `retry #${i} should have been scheduled`).not.toBeNull()
+ expect(entry.p2pRetryCount).toBe(i)
+ vi.advanceTimersByTime(60_000)
+ entry.p2pRetryTimer = null // the real timer callback nulls this before re-dialing; simulate that
+ }
+ pool.scheduleP2pRetry('machine-1', entry) // one retry past the cap — refused, quota spent
+ expect(entry.p2pRetryTimer).toBeNull()
+ expect(entry.p2pRetryCount).toBe(RETRY_MAX)
+ } finally {
+ vi.useRealTimers()
+ }
+ })
+
+ it('does not double-schedule while a retry timer is already pending', () => {
+ vi.useFakeTimers()
+ try {
+ const entry = fakeEntry()
+ pool.scheduleP2pRetry('machine-1', entry)
+ const firstTimer = entry.p2pRetryTimer
+ pool.scheduleP2pRetry('machine-1', entry) // a second failure before the first retry even fires
+ expect(entry.p2pRetryTimer).toBe(firstTimer)
+ expect(entry.p2pRetryCount).toBe(1)
+ } finally {
+ vi.useRealTimers()
+ }
+ })
+
+ it('a success resets the short-term counter so a later demote gets a fresh budget', () => {
+ const entry = fakeEntry({ p2pRetryCount: 3 }) // quota exhausted from a prior run of failures
+ entry.p2pRetryCount = 0 // what startP2p's onState does on 'direct' — exercised here directly
+ pool.scheduleP2pRetry('machine-1', entry)
+ expect(entry.p2pRetryTimer).not.toBeNull()
+ expect(entry.p2pRetryCount).toBe(1)
+ })
+
+ it('the lifetime cap stops scheduling even with a fresh short-term counter', () => {
+ const entry = fakeEntry({ p2pRetryLifetimeTotal: 10 })
+ pool.scheduleP2pRetry('machine-1', entry)
+ expect(entry.p2pRetryTimer).toBeNull()
+ expect(entry.p2pRetryCount).toBe(0)
+ })
+
+ it('demoteP2p (case B: was direct, then demoted) schedules a retry through the same policy', () => {
+ const entry = fakeEntry({ p2pStreams: new Set(['stream-a']) })
+ pool.demoteP2p('machine-1', entry, 'send_failed')
+ expect(entry.p2pRetryTimer).not.toBeNull()
+ expect(entry.p2pRetryCount).toBe(1)
+ })
+})
+
+describe('RemoteRelayPool live-migration of already-open streams onto p2p', () => {
+ const pool = new RemoteRelayPool(
+ { accessToken: async () => 'unused' } as never,
+ 'ws://unused',
+ { pub: new Uint8Array(), priv: new Uint8Array() } as never,
+ { pin: () => {}, get: () => null } as never,
+ ) as unknown as {
+ promoteOpenStreams: (machineId: string, entry: ReturnType<typeof fakeEntry>) => void
+ commitMigration: (machineId: string, entry: ReturnType<typeof fakeEntry>, streamId: string) => void
+ noteTerminalResponse: (entry: ReturnType<typeof fakeEntry>, frame: Record<string, unknown>, transport: 'p2p' | 'relay') => void
+ }
+
+ it('phase 1 sends terminal_resync over the CURRENT transport (relay) for every open, not-yet-p2p stream', () => {
+ const entry = fakeEntry({ streams: new Set(['s1', 's2']), p2pStreams: new Set(['s2']) })
+ pool.promoteOpenStreams('machine-1', entry)
+
+ // s2 is already p2p — skipped. Only s1 (open, on relay) becomes a migration candidate.
+ expect(entry.p2pMigrating.has('s1')).toBe(true)
+ expect(entry.p2pMigrating.has('s2')).toBe(false)
+ expect(entry.ws.send).toHaveBeenCalledWith(JSON.stringify({ type: 'terminal_resync', payload: { streamId: 's1' } }))
+ })
+
+ it('a stream already migrating or already p2p is not re-queued', () => {
+ const entry = fakeEntry({ streams: new Set(['s1']), p2pMigrating: new Map([['s1', Date.now()]]) })
+ pool.promoteOpenStreams('machine-1', entry)
+ expect(entry.ws.send).not.toHaveBeenCalled()
+ })
+
+ it('phase 2 (commitMigration) sends a second resync over p2p and flips p2pStreams, but keeps p2pMigrating until confirmed', () => {
+ const p2pSend = vi.fn(() => true)
+ const entry = fakeEntry({
+ streams: new Set(['s1']),
+ p2pMigrating: new Map([['s1', Date.now()]]),
+ p2p: { isReady: true, send: p2pSend },
+ })
+ pool.commitMigration('machine-1', entry, 's1')
+
+ expect(entry.p2pStreams.has('s1')).toBe(true)
+ expect(p2pSend).toHaveBeenCalledWith(JSON.stringify({ type: 'terminal_resync', payload: { streamId: 's1' } }))
+ expect(entry.p2pMigrating.has('s1')).toBe(true) // not confirmed yet — see next test
+ })
+
+ it('confirmation (a p2p-delivered frame for the stream) clears p2pMigrating and re-arms the demote-safety net', () => {
+ const entry = fakeEntry({
+ p2pStreams: new Set(['s1']),
+ p2pMigrating: new Map([['s1', Date.now()]]),
+ })
+ pool.noteTerminalResponse(entry, { type: 'terminal_alive', payload: { streamId: 's1' } }, 'p2p')
+ expect(entry.p2pMigrating.has('s1')).toBe(false)
+ })
+
+ it('commitMigration abandons just this stream if p2p is not ready, without touching other state', () => {
+ const entry = fakeEntry({ streams: new Set(['s1']), p2pMigrating: new Map([['s1', Date.now()]]), p2p: null })
+ pool.commitMigration('machine-1', entry, 's1')
+
+ expect(entry.p2pStreams.has('s1')).toBe(false)
+ expect(entry.p2pMigrating.has('s1')).toBe(false)
+ })
+
+ it('a relay frame for a stream mid-migration does NOT trigger the demote-on-mismatch rule', () => {
+ const entry = fakeEntry({
+ p2pStreams: new Set(['s1']),
+ p2pMigrating: new Map([['s1', Date.now()]]),
+ })
+ pool.noteTerminalResponse(entry, { type: 'terminal_alive', payload: { streamId: 's1' } }, 'relay')
+
+ // Without the p2pMigrating guard this would have deleted s1 from p2pStreams and reported 'relay'.
+ expect(entry.p2pStreams.has('s1')).toBe(true)
+ expect(entry.sink.sendFrame).not.toHaveBeenCalled()
+ })
+})
diff --git a/cli/src/lib/remoteRelay.ts b/cli/src/lib/remoteRelay.ts
index 66a0647d4..0027c8c42 100644
--- a/cli/src/lib/remoteRelay.ts
+++ b/cli/src/lib/remoteRelay.ts
@@ -40,6 +40,19 @@ const LINGER_MS = 30_000
// redial. `ws.terminate()` on a missed pong forces the existing `close` handler to run cleanup.
const HEARTBEAT_MS = 20_000
+// A p2p attempt that never reaches 'direct', or one that did and then got demoted mid-session, both
+// leave the stream(s) on the ws relay with nothing to bring them back — see scheduleP2pRetry(). One
+// policy for both cases: retry every 60s, capped at 3 retries (4 attempts total) since the last
+// success, reset on the next 'direct'. p2pRetryLifetimeTotal is the belt-and-braces cap across the
+// entry's whole life, so a connection that flaps direct/demoted forever every few minutes cannot
+// retry forever just because each success resets the short-term counter.
+const P2P_RETRY_DELAY_MS = 60_000
+const P2P_RETRY_MAX = 3
+const P2P_RETRY_LIFETIME_CAP = 10
+// How long a stream may sit in p2pMigrating before the sweep (piggybacked on heartbeatTimer) gives up
+// on it and lets the ordinary demote-on-mismatch rule apply again.
+const P2P_MIGRATION_TTL_MS = 30_000
+
export class RelayConnectError extends Error {
constructor(message: string, readonly closeCode?: number) {
super(message)
@@ -66,6 +79,17 @@ interface Entry {
p2pPolicy: TerminalP2pPolicy | null
p2pPendingOpens: Set<string>
p2pStreams: Set<string>
+ /** Every streamId currently open on this entry, regardless of transport — the superset p2pStreams
+ * is drawn from, and what promoteOpenStreams() walks to find migration candidates. */
+ streams: Set<string>
+ /** streamId -> when its migration to p2p started, so a stuck one can be swept off the heartbeat. */
+ p2pMigrating: Map<string, number>
+ /** Retries since the last 'direct' (or since entry creation, if never direct). Reset to 0 on 'direct'. */
+ p2pRetryCount: number
+ /** Retries across this entry's whole life — never reset. The safeguard against a flapping connection
+ * retrying forever purely because each success resets p2pRetryCount. */
+ p2pRetryLifetimeTotal: number
+ p2pRetryTimer: ReturnType<typeof setTimeout> | null
}
export interface RelaySession {
@@ -123,6 +147,7 @@ export class RemoteRelayPool {
this.entries.delete(machineId)
if (entry.heartbeatTimer) clearInterval(entry.heartbeatTimer)
if (entry.lingerTimer) clearTimeout(entry.lingerTimer)
+ if (entry.p2pRetryTimer) clearTimeout(entry.p2pRetryTimer)
// The caller is invalidating so it can immediately acquire() a fresh entry on the SAME local
// connection (it just got a forceReconnect select) — null this out first so the generic
// `ws.on('close', ...)` cleanup below doesn't turn around and close that same local socket via a
@@ -202,6 +227,11 @@ export class RemoteRelayPool {
p2pPolicy: null,
p2pPendingOpens: new Set(),
p2pStreams: new Set(),
+ streams: new Set(),
+ p2pMigrating: new Map(),
+ p2pRetryCount: 0,
+ p2pRetryLifetimeTotal: 0,
+ p2pRetryTimer: null,
}
// Two phases before this connection is usable: (1) machine_select ack, (2) this daemon's own
// e2e_hello/e2e_welcome as the "client" role — see lib/e2ee/relayClient.ts. Only once BOTH are done
@@ -295,7 +325,17 @@ export class RemoteRelayPool {
if (isBinary) {
const clear = crypto.decryptTerminal(binaryBytes(raw))
if (!clear) return // undecryptable/stale — drop, never forward ciphertext or garbage to the app
- if (entry.p2pStreams.has(clear.streamId)) this.demoteP2p(entry, 'relay_binary_received')
+ if (clear.kind === TerminalBinaryKind.keyframe && entry.p2pMigrating.has(clear.streamId)
+ && !entry.p2pStreams.has(clear.streamId)) {
+ // Phase 1 of a live migration completing: the responder's reply to our WS-side terminal_resync
+ // proves it has drained/snapshotted this stream as of now, so it is safe to trigger phase 2.
+ this.commitMigration(machineId, entry, clear.streamId)
+ } else if (entry.p2pStreams.has(clear.streamId) && !entry.p2pMigrating.has(clear.streamId)) {
+ // Suppressed while p2pMigrating holds this streamId: during phase 2 the responder may still
+ // legitimately emit relay-routed output for it until ITS OWN flip lands, and that must not be
+ // read as p2p having broken.
+ this.demoteP2p(machineId, entry, 'relay_binary_received')
+ }
const local = encodeTerminalLocal(clear)
if (local) entry.sink?.sendBinary(local)
return
@@ -341,6 +381,7 @@ export class RemoteRelayPool {
// Handshake done — from here on, a close is the entry's real end-of-life, not a handshake failure.
ws.on('close', (code, reasonBuf) => {
if (entry.heartbeatTimer) clearInterval(entry.heartbeatTimer)
+ if (entry.p2pRetryTimer) clearTimeout(entry.p2pRetryTimer)
void entry.p2p?.stop('relay_closed', false)
entry.p2p = null
this.entries.delete(machineId)
@@ -351,6 +392,14 @@ export class RemoteRelayPool {
if (!entry.alive) { ws.terminate(); return }
entry.alive = false
try { ws.ping() } catch { ws.terminate() }
+ // Piggybacked sweep for a migration that never completed (pane closed mid-flight, responder never
+ // answered, etc.) — no dedicated timer needed, this tick is frequent enough (20s) against the 30s TTL.
+ if (entry.p2pMigrating.size > 0) {
+ const cutoff = Date.now() - P2P_MIGRATION_TTL_MS
+ for (const [streamId, startedAt] of entry.p2pMigrating) {
+ if (startedAt < cutoff) entry.p2pMigrating.delete(streamId)
+ }
+ }
}, HEARTBEAT_MS)
entry.heartbeatTimer.unref?.()
this.entries.set(machineId, entry)
@@ -368,12 +417,23 @@ export class RemoteRelayPool {
else this.reportP2pResult(entry, 'relay', undefined, 'open_wait_elapsed')
}
const wrapped = entry.crypto.wrapOutgoing(frame)
+ const closingStreamId = frame.type === 'terminal_close' && typeof payload.streamId === 'string'
+ ? payload.streamId
+ : null
if (useP2p && entry.p2p?.send(JSON.stringify(wrapped))) {
- if (frame.type === 'terminal_close' && typeof payload.streamId === 'string') entry.p2pStreams.delete(payload.streamId)
+ if (closingStreamId) {
+ entry.p2pStreams.delete(closingStreamId)
+ entry.streams.delete(closingStreamId)
+ entry.p2pMigrating.delete(closingStreamId)
+ }
return
}
try { entry.ws.send(JSON.stringify(wrapped)) } catch { /* closed — onClosed will fire */ }
- if (useP2p) this.demoteP2p(entry, 'send_failed')
+ if (useP2p) this.demoteP2p(machineId, entry, 'send_failed')
+ if (closingStreamId) {
+ entry.streams.delete(closingStreamId)
+ entry.p2pMigrating.delete(closingStreamId)
+ }
},
sendBinary: async (clear) => {
const sealed = entry.crypto.encryptTerminal(clear)
@@ -381,13 +441,14 @@ export class RemoteRelayPool {
if (entry.p2pStreams.has(clear.streamId) && entry.p2p?.send(Buffer.from(sealed))) return
const p2pFailed = entry.p2pStreams.has(clear.streamId)
try { entry.ws.send(sealed, { binary: true }) } catch { /* closed — onClosed will fire */ }
- if (p2pFailed) this.demoteP2p(entry, 'send_failed')
+ if (p2pFailed) this.demoteP2p(machineId, entry, 'send_failed')
},
detach: () => {
entry.sink = null
entry.onClosed = null
entry.lingerTimer = setTimeout(() => {
if (!entry.sink) {
+ if (entry.p2pRetryTimer) clearTimeout(entry.p2pRetryTimer)
void entry.p2p?.stop('idle', false)
entry.p2p = null
try { entry.ws.close(1000, 'idle') } catch { /* ignore */ }
@@ -420,18 +481,25 @@ export class RemoteRelayPool {
onState: (state, setupMs, reason) => {
if (state === 'direct') {
wasDirect = true
+ entry.p2pRetryCount = 0 // a fresh success re-arms the full retry budget for the NEXT demote, if any
// 'relayed' distinguishes a Cloudflare TURN path from a truly direct one. Both are "p2p" as far
// as the terminal is concerned, but only one of them is billed per GB.
const relayed = p2p.transport === 'relay'
console.log(`[p2p] connected · machine=${sid(machineId)} via=${relayed ? 'turn' : 'direct'} setup=${Math.round(setupMs)}ms`)
this.reportP2pResult(entry, 'direct', setupMs, relayed ? 'relayed' : undefined)
+ this.promoteOpenStreams(machineId, entry)
} else if (state === 'failed' && !wasDirect) {
console.log(`[p2p] gave up · machine=${sid(machineId)} reason=${reason ?? 'unknown'} after=${Math.round(setupMs)}ms`
+ ` · ${p2p.negotiationDetail} — terminals stay on the ws relay`)
this.reportP2pResult(entry, reason === 'negotiation_timeout' ? 'timeout' : 'failed', setupMs, reason)
+ // Without this the entry is stuck: entry.p2p still points at this now-finished instance, and
+ // startP2p()'s own guard (`if (!policy || entry.p2p) return`) would block every future attempt
+ // forever — previously the only way out was a full entry teardown (network drop).
+ entry.p2p = null
+ this.scheduleP2pRetry(machineId, entry)
}
},
- onUnavailable: (reason) => this.demoteP2p(entry, reason),
+ onUnavailable: (reason) => this.demoteP2p(machineId, entry, reason),
})
entry.p2p = p2p
p2p.start()
@@ -443,6 +511,10 @@ export class RemoteRelayPool {
if (!clear) return
if (clear.kind !== TerminalBinaryKind.output && clear.kind !== TerminalBinaryKind.keyframe
&& clear.kind !== TerminalBinaryKind.sync) return
+ // Phase 2 of a live migration confirmed: the responder has committed its own flip and this
+ // stream's bytes are now genuinely arriving over p2p — safe to re-arm the ordinary
+ // demote-on-mismatch rule for it (see the guard in the ws binary handler above).
+ entry.p2pMigrating.delete(clear.streamId)
const local = encodeTerminalLocal(clear)
if (local) entry.sink?.sendBinary(local)
return
@@ -479,7 +551,11 @@ export class RemoteRelayPool {
const payload = framePayload(frame)
const streamId = typeof payload.streamId === 'string' ? payload.streamId : ''
const requestId = typeof payload.requestId === 'string' ? payload.requestId : ''
+ // Any p2p-delivered frame for a stream still being migrated is itself proof the responder has
+ // committed its flip — re-arm the ordinary demote-on-mismatch rule below for it.
+ if (transport === 'p2p' && streamId) entry.p2pMigrating.delete(streamId)
if (frame.type === 'terminal_ready' && requestId && streamId) {
+ entry.streams.add(streamId) // the full "open, any transport" registry promoteOpenStreams() walks
if (entry.p2pPendingOpens.delete(requestId) && transport === 'p2p') entry.p2pStreams.add(streamId)
// Tell the local app (Desktop) which transport this stream just came up on — derived from
// entry.p2pStreams' own post-update membership (the routing source of truth just above), not a
@@ -490,28 +566,94 @@ export class RemoteRelayPool {
entry.p2pPendingOpens.delete(requestId)
} else if (frame.type === 'terminal_closed' && streamId) {
entry.p2pStreams.delete(streamId)
+ entry.streams.delete(streamId)
+ entry.p2pMigrating.delete(streamId)
}
// A non-terminal_ready frame for a stream still marked p2p but physically delivered over relay: a
// quieter, single-stream demotion than demoteP2p() (which also tears down the whole p2p connection)
- // — still worth telling the local app about, since its badge would otherwise go stale.
- if (transport === 'relay' && streamId && entry.p2pStreams.delete(streamId)) {
+ // — still worth telling the local app about, since its badge would otherwise go stale. Suppressed
+ // while p2pMigrating holds this streamId — see the ws binary handler's matching guard.
+ if (transport === 'relay' && streamId && !entry.p2pMigrating.has(streamId) && entry.p2pStreams.delete(streamId)) {
entry.sink?.sendFrame({ type: 'terminal_link_mode', payload: { streamId, mode: 'relay' } })
}
}
- private demoteP2p(entry: Entry, reason: string): void {
+ private demoteP2p(machineId: string, entry: Entry, reason: string): void {
const p2p = entry.p2p
entry.p2p = null
const streamIds = [...entry.p2pStreams]
entry.p2pStreams.clear()
entry.p2pPendingOpens.clear()
for (const streamId of streamIds) {
+ entry.p2pMigrating.delete(streamId)
const resync = entry.crypto.wrapOutgoing({ type: 'terminal_resync', payload: { streamId } })
try { entry.ws.send(JSON.stringify(resync)) } catch { /* relay close handles cleanup */ }
entry.sink?.sendFrame({ type: 'terminal_link_mode', payload: { streamId, mode: 'relay' } })
}
if (streamIds.length > 0) this.reportP2pResult(entry, 'dropped', undefined, reason)
void p2p?.stop(reason)
+ this.scheduleP2pRetry(machineId, entry)
+ }
+
+ /**
+ * One retry policy for both ways an entry ends up needing p2p back: never reached 'direct' at all,
+ * or reached it once and then got demoted. Either way `entry.p2p` is null here (the failed/demoted
+ * instance already cleared it) so `startP2p`'s own guard will accept the retry once the timer fires.
+ *
+ * p2pRetryCount is reset to 0 on every 'direct' (see startP2p's onState), so a demote long after a
+ * clean run gets the full budget again — p2pRetryLifetimeTotal, which never resets, is what stops a
+ * connection that flaps direct/demoted forever from retrying forever.
+ */
+ private scheduleP2pRetry(machineId: string, entry: Entry): void {
+ if (entry.p2pRetryTimer) return
+ if (entry.p2pRetryLifetimeTotal >= P2P_RETRY_LIFETIME_CAP) return
+ if (entry.p2pRetryCount >= P2P_RETRY_MAX) return
+ entry.p2pRetryCount++
+ entry.p2pRetryLifetimeTotal++
+ entry.p2pRetryTimer = setTimeout(() => {
+ entry.p2pRetryTimer = null
+ if (this.entries.get(machineId) !== entry) return // entry was torn down/replaced meanwhile
+ entry.p2p = null
+ this.startP2p(machineId, entry)
+ }, P2P_RETRY_DELAY_MS)
+ entry.p2pRetryTimer.unref?.()
+ }
+
+ /**
+ * Migrate every stream already open on this entry (any transport) onto p2p, once it reaches
+ * 'direct' — whether that is the very first success or a later retry succeeding. Two phases, both
+ * reusing the existing terminal_resync frame rather than inventing a new signal:
+ *
+ * 1. Here: send terminal_resync over whichever transport the stream is CURRENTLY on (relay, since
+ * it is not yet in p2pStreams) — a drain barrier. The responder answers with a fresh keyframe
+ * over that same relay path once it has processed everything before this point.
+ * 2. commitMigration(): once that keyframe arrives back over relay, THAT is the proof the drain
+ * landed, and only then do we flip this side to p2p and send a second terminal_resync — this
+ * time over p2p — which is what makes the responder flip its own routing table too.
+ *
+ * Never both transports live for the same direction at once: input keeps going over relay until
+ * commitMigration flips p2pStreams, and output only starts riding p2p once the responder's own flip
+ * (triggered by that second resync) lands.
+ */
+ private promoteOpenStreams(machineId: string, entry: Entry): void {
+ for (const streamId of entry.streams) {
+ if (entry.p2pStreams.has(streamId) || entry.p2pMigrating.has(streamId)) continue
+ entry.p2pMigrating.set(streamId, Date.now())
+ void this.sessionFor(machineId, entry).send({ type: 'terminal_resync', payload: { streamId } })
+ }
+ }
+
+ /** Phase 2 of promoteOpenStreams() — see its doc comment. Sends directly on entry.p2p rather than
+ * through sessionFor().send(): a failure here should only abandon THIS stream's migration, not tear
+ * down the whole p2p connection the way sessionFor's own failure handling would. */
+ private commitMigration(machineId: string, entry: Entry, streamId: string): void {
+ if (!entry.p2p?.isReady) { entry.p2pMigrating.delete(streamId); return }
+ const wrapped = entry.crypto.wrapOutgoing({ type: 'terminal_resync', payload: { streamId } })
+ if (!entry.p2p.send(JSON.stringify(wrapped))) { entry.p2pMigrating.delete(streamId); return }
+ entry.p2pStreams.add(streamId)
+ // p2pMigrating is deliberately NOT cleared yet — that happens only once a p2p-delivered frame for
+ // this stream actually arrives (handleP2pData / noteTerminalResponse), which is the real proof the
+ // responder committed its own flip. Until then the demote-on-mismatch rule stays suppressed for it.
}
private reportP2pResult(entry: Entry, outcome: string, setupMs?: number, reason?: string): void {
diff --git a/cli/src/lib/terminalStreamManager.ts b/cli/src/lib/terminalStreamManager.ts
index f4093ff9f..9f32041bb 100644
--- a/cli/src/lib/terminalStreamManager.ts
+++ b/cli/src/lib/terminalStreamManager.ts
@@ -344,6 +344,15 @@ export class TerminalStreamManager {
return state?.connId === connId && !state.closing ? state : null
}
+ /** Existence/ownership check for connId+streamId — used by BackendSocket to validate a p2p-arrived
+ * terminal_resync (a live-migration promotion signal, not an actual resync request) actually names a
+ * live stream owned by this connection before trusting it, instead of leaking a routing entry for a
+ * stale/bogus streamId (e.g. the pane closed in the same instant the migration barrier was in flight). */
+ hasStream(connId: string, streamId: string): boolean {
+ const state = this.streams.get(streamId)
+ return !!state && state.connId === connId && !state.closing
+ }
+
private alive(connId: string, payload: FramePayload): void {
const state = this.streamFor(connId, payload)
if (!state) return