Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

Implement backpressure handling for P2P terminal streams by adding a retriable send method with buffered-amount monitoring.

0xken154a48d10d8cf7
diff --git a/cli/src/lib/remoteRelay.spec.ts b/cli/src/lib/remoteRelay.spec.ts
index 689804e66..20201916d 100644
--- a/cli/src/lib/remoteRelay.spec.ts
+++ b/cli/src/lib/remoteRelay.spec.ts
@@ -1,4 +1,4 @@
-import { describe, expect, it, vi } from 'vitest'
+import { afterEach, describe, expect, it, vi } from 'vitest'
import { RemoteRelayPool } from './remoteRelay.js'
/** Minimal fake of the private `Entry` shape `noteTerminalResponse`/`demoteP2p` operate on — exercised
@@ -8,7 +8,7 @@ import { RemoteRelayPool } from './remoteRelay.js'
function fakeEntry(overrides: Partial<Record<string, unknown>> = {}) {
return {
ws: { send: vi.fn() },
- crypto: { wrapOutgoing: (frame: unknown) => frame },
+ crypto: { wrapOutgoing: (frame: unknown) => frame, encryptTerminal: () => new Uint8Array([1, 2, 3]) },
sink: { sendFrame: vi.fn(() => true), sendBinary: vi.fn(() => true) },
onClosed: null,
lingerTimer: null,
@@ -301,6 +301,7 @@ function fakeP2p(overrides: Partial<Record<string, unknown>> = {}) {
isReady: true,
transport: 'direct',
send: vi.fn(() => true),
+ sendWithBackpressureRetry: vi.fn(async () => true),
stop: vi.fn(async () => {}),
...overrides,
}
@@ -450,3 +451,61 @@ describe('RemoteRelayPool TURN-to-direct upgrade', () => {
expect(entry.upgradeTimer).toBeNull()
})
})
+
+// Regression: a burst of large frames (a chunked upload's chunks, fired back-to-back) used to cross
+// TERMINAL_P2P_MAX_BUFFERED_BYTES well before the real network drained the backlog, and a single
+// resulting `send()` failure was misread as the whole p2p connection being dead — tearing it down for
+// every stream, not just the one that hit backpressure. sendBinary must go through
+// sendWithBackpressureRetry (which gives the channel one bounded chance to drain) rather than calling
+// `send` directly, and must only fall back to relay + demote once that retry itself fails.
+describe('RemoteRelayPool sendBinary backpressure handling', () => {
+ const pool = new RemoteRelayPool(
+ { accessToken: async () => 'unused' } as never,
+ 'ws://unused',
+ { pub: new Uint8Array(), priv: new Uint8Array() } as never,
+ { pin: () => {}, get: () => null } as never,
+ ) as unknown as {
+ sessionFor: (machineId: string, entry: ReturnType<typeof fakeEntry>) => { sendBinary: (clear: { streamId: string }) => Promise<void> }
+ demoteP2p: (machineId: string, entry: ReturnType<typeof fakeEntry>, reason: string) => void
+ }
+
+ // demoteP2p's own real behavior (resync-per-stream over ws, scheduleP2pRetry's real setTimeout) is
+ // covered by the dedicated describe blocks above/below — stubbed out here to a no-op so these tests
+ // observe only sendBinary's own routing decision, not demoteP2p's side effects or timers leaking
+ // between tests.
+ afterEach(() => { vi.restoreAllMocks() })
+
+ it('sends over p2p via sendWithBackpressureRetry, not send, and never touches ws or demotes on success', async () => {
+ const p2p = fakeP2p()
+ const entry = fakeEntry({ p2p, p2pStreams: new Set(['stream-1']) })
+ vi.spyOn(pool, 'demoteP2p').mockImplementation(() => {})
+ await pool.sessionFor('machine-1', entry).sendBinary({ streamId: 'stream-1' })
+
+ expect(p2p.sendWithBackpressureRetry).toHaveBeenCalledTimes(1)
+ expect(p2p.send).not.toHaveBeenCalled()
+ expect(entry.ws.send).not.toHaveBeenCalled()
+ expect(pool.demoteP2p).not.toHaveBeenCalled()
+ })
+
+ it('falls back to ws and demotes only once sendWithBackpressureRetry itself fails', async () => {
+ const p2p = fakeP2p({ sendWithBackpressureRetry: vi.fn(async () => false) })
+ const entry = fakeEntry({ p2p, p2pStreams: new Set(['stream-1']) })
+ const demoteP2p = vi.spyOn(pool, 'demoteP2p').mockImplementation(() => {})
+ await pool.sessionFor('machine-1', entry).sendBinary({ streamId: 'stream-1' })
+
+ expect(p2p.sendWithBackpressureRetry).toHaveBeenCalledTimes(1)
+ expect(entry.ws.send).toHaveBeenCalledTimes(1) // the chunk still gets there, just over ws
+ expect(demoteP2p).toHaveBeenCalledWith('machine-1', entry, 'send_failed')
+ })
+
+ it('goes straight to ws with no demotion for a stream that was never on p2p', async () => {
+ const p2p = fakeP2p()
+ const entry = fakeEntry({ p2p, p2pStreams: new Set() }) // this stream never made it onto p2pStreams
+ const demoteP2p = vi.spyOn(pool, 'demoteP2p').mockImplementation(() => {})
+ await pool.sessionFor('machine-1', entry).sendBinary({ streamId: 'stream-1' })
+
+ expect(p2p.sendWithBackpressureRetry).not.toHaveBeenCalled()
+ expect(entry.ws.send).toHaveBeenCalledTimes(1)
+ expect(demoteP2p).not.toHaveBeenCalled()
+ })
+})
diff --git a/cli/src/lib/remoteRelay.ts b/cli/src/lib/remoteRelay.ts
index 652c6fba1..0f2769e20 100644
--- a/cli/src/lib/remoteRelay.ts
+++ b/cli/src/lib/remoteRelay.ts
@@ -504,8 +504,21 @@ export class RemoteRelayPool {
sendBinary: async (clear) => {
const sealed = entry.crypto.encryptTerminal(clear)
if (!sealed) return
- if (entry.p2pStreams.has(clear.streamId) && entry.p2p?.send(Buffer.from(sealed))) return
- const p2pFailed = entry.p2pStreams.has(clear.streamId)
+ const wantsP2p = entry.p2pStreams.has(clear.streamId)
+ // A burst of large frames (a chunked upload's chunks, fired back-to-back) can push the data
+ // channel's send buffer over TERMINAL_P2P_MAX_BUFFERED_BYTES well before the real network has
+ // drained it — sendWithBackpressureRetry gives that ONE bounded chance to clear before this
+ // falls through to relay-and-demote, instead of misreading a busy channel as a dead one. A
+ // plain successful send resolves in microseconds, so timing the call is enough to tell "it had
+ // to wait" apart from the ordinary case without threading a reporting hook through the p2p class.
+ const startedAt = Date.now()
+ const p2pSent = wantsP2p && await entry.p2p?.sendWithBackpressureRetry(Buffer.from(sealed))
+ const waited = Date.now() - startedAt >= 50
+ if (waited) {
+ console.log(`[p2p] backpressure · machine=${sid(machineId)} drained=${p2pSent ? 'yes' : 'no'} after=${Date.now() - startedAt}ms`)
+ }
+ if (p2pSent) return
+ const p2pFailed = wantsP2p
try { entry.ws.send(sealed, { binary: true }) } catch { /* closed — onClosed will fire */ }
if (p2pFailed) this.demoteP2p(machineId, entry, 'send_failed')
},
diff --git a/cli/src/lib/terminalP2p.spec.ts b/cli/src/lib/terminalP2p.spec.ts
index ad4024747..8b2ccd254 100644
--- a/cli/src/lib/terminalP2p.spec.ts
+++ b/cli/src/lib/terminalP2p.spec.ts
@@ -7,10 +7,26 @@ import {
TERMINAL_P2P_UP_TYPES,
TerminalP2pInitiator,
TerminalP2pResponderPool,
+ waitForBufferedAmountLow,
type TerminalP2pSignal,
} from './terminalP2p.js'
import type { StunSelector } from './stunSelect.js'
+/** Structurally shaped like the slice of werift's `RTCDataChannel` this function actually reads —
+ * no real WebRTC negotiation needed, same reasoning as `isRelayedPair`'s plain-string fixtures. */
+function fakeChannel(initial: {
+ readyState?: 'open' | 'closed' | 'connecting' | 'closing'
+ bufferedAmount?: number
+ asPromise?: () => Promise<unknown[]>
+}) {
+ return {
+ readyState: initial.readyState ?? 'open',
+ bufferedAmount: initial.bufferedAmount ?? 0,
+ bufferedAmountLow: { asPromise: initial.asPromise ?? (() => new Promise<unknown[]>(() => {})) },
+ }
+}
+type FakeChannel = ReturnType<typeof fakeChannel>
+
describe('terminal WebRTC data channel', () => {
it('negotiates locally and carries ordered text and binary frames in both directions', async () => {
let resolveResponderData!: (value: string) => void
@@ -250,6 +266,43 @@ describe('terminal WebRTC data channel', () => {
expect(isRelayedPair('candidate:1 1 udp 1 10.0.0.1 1 typ host raddr relay.example')).toBe(false)
})
+ // Regression: a burst of sends (a chunked upload's chunks, fired back-to-back) can cross
+ // TERMINAL_P2P_MAX_BUFFERED_BYTES well before the real network drains the backlog — this is the
+ // wait this puts to use instead of every send() in that window being mistaken for a dead channel.
+ describe('waitForBufferedAmountLow', () => {
+ const wait = (channel: FakeChannel, timeoutMs = 1_000) =>
+ waitForBufferedAmountLow(channel as unknown as Parameters<typeof waitForBufferedAmountLow>[0], timeoutMs)
+
+ it('resolves immediately when the buffer is already under the ceiling', async () => {
+ const channel = fakeChannel({ bufferedAmount: 0 })
+ await expect(wait(channel)).resolves.toBe(true)
+ })
+
+ it('fails fast with no wait when the channel is not open', async () => {
+ const asPromise = vi.fn(() => new Promise<unknown[]>(() => {}))
+ const channel = fakeChannel({ readyState: 'closed', bufferedAmount: 5_000_000, asPromise })
+ await expect(wait(channel)).resolves.toBe(false)
+ expect(asPromise).not.toHaveBeenCalled() // a dead channel is never worth waiting on
+ })
+
+ it('waits for the drain event, then re-checks the buffer before resolving true', async () => {
+ const channel = fakeChannel({ bufferedAmount: 5_000_000, asPromise: () => Promise.resolve([]) })
+ const result = wait(channel)
+ channel.bufferedAmount = 0 // the buffer clears right as the event fires
+ await expect(result).resolves.toBe(true)
+ })
+
+ it('resolves false if the buffer is still over the ceiling once the event fires', async () => {
+ const channel = fakeChannel({ bufferedAmount: 5_000_000, asPromise: () => Promise.resolve([]) })
+ await expect(wait(channel)).resolves.toBe(false) // bufferedAmount never actually dropped
+ })
+
+ it('resolves false when the wait itself times out', async () => {
+ const channel = fakeChannel({ bufferedAmount: 5_000_000, asPromise: () => Promise.reject(new Error('timeout')) })
+ await expect(wait(channel)).resolves.toBe(false)
+ })
+ })
+
// Regression: a JSON type these two allowlists don't know about is silently dropped the instant a
// pane's connection is on a direct p2p data channel (backendSocket.ts's `handleP2pData` / remoteRelay.ts's
// p2p-data handler) — no error, no log, indistinguishable from the wire itself hanging. That's exactly
diff --git a/cli/src/lib/terminalP2p.ts b/cli/src/lib/terminalP2p.ts
index 40856b1dd..683c70eb4 100644
--- a/cli/src/lib/terminalP2p.ts
+++ b/cli/src/lib/terminalP2p.ts
@@ -189,6 +189,27 @@ function channelCanSend(channel: RTCDataChannel | null): channel is RTCDataChann
&& channel.bufferedAmount < TERMINAL_P2P_MAX_BUFFERED_BYTES
}
+/**
+ * Waits for `channel`'s send buffer to drop back under the ceiling, using the
+ * `bufferedAmountLowThreshold`/`bufferedAmountLow` plumbing this file already configures
+ * (`wireChannel`/`wireResponderEntry` below) but never previously consulted — a burst of sends (a
+ * chunked upload's chunks, fired back-to-back) can cross `TERMINAL_P2P_MAX_BUFFERED_BYTES` well
+ * before the real network has drained the backlog, and without this wait every `send()` call in that
+ * window was mistaken for the channel being dead outright. Resolves `false` on a genuine channel
+ * closure or if the buffer hasn't cleared within `timeoutMs` — either way, the caller's existing
+ * fall-back-to-relay-and-demote behavior is exactly what should happen next.
+ */
+export async function waitForBufferedAmountLow(channel: RTCDataChannel, timeoutMs: number): Promise<boolean> {
+ if (channel.readyState !== 'open') return false
+ if (channel.bufferedAmount < TERMINAL_P2P_MAX_BUFFERED_BYTES) return true
+ try {
+ await channel.bufferedAmountLow.asPromise(timeoutMs)
+ } catch {
+ return false // timed out, or the channel errored while we waited
+ }
+ return channel.readyState === 'open' && channel.bufferedAmount < TERMINAL_P2P_MAX_BUFFERED_BYTES
+}
+
/** Source side: owns the offerer for one pooled remote-machine relay connection. */
export class TerminalP2pInitiator {
readonly sessionId = randomUUID()
@@ -343,6 +364,23 @@ export class TerminalP2pInitiator {
}
}
+ /**
+ * Same contract as `send()`, but a failure caused purely by backpressure (the channel is still
+ * `open`, just over `TERMINAL_P2P_MAX_BUFFERED_BYTES` right now) gets ONE bounded chance to drain
+ * before being reported as failure — see `waitForBufferedAmountLow`'s own comment for why this
+ * exists. A `send()` failure for any other reason (channel not open/ready, or `channel.send()`
+ * itself threw, which schedules `fail()`) is NOT retried: re-reading `channel.readyState` fresh
+ * rather than trusting `this.isReady` is what tells the two apart, since a queued `fail()` from a
+ * hard send error hasn't necessarily flipped `this.ready` by the time this runs.
+ */
+ async sendWithBackpressureRetry(data: TerminalP2pData, drainTimeoutMs = 5_000): Promise<boolean> {
+ if (this.send(data)) return true
+ const channel = this.channel
+ if (!channel || channel.readyState !== 'open') return false
+ if (!(await waitForBufferedAmountLow(channel, drainTimeoutMs))) return false
+ return this.send(data)
+ }
+
waitUntilReady(timeoutMs: number): Promise<boolean> {
if (this.isReady) return Promise.resolve(true)
if (this.finished || timeoutMs <= 0) return Promise.resolve(false)