The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
Implement backpressure handling for P2P terminal streams by adding a retriable send method with buffered-amount monitoring.
0xken154a48d10d8cf7
diff --git a/cli/src/lib/remoteRelay.spec.ts b/cli/src/lib/remoteRelay.spec.tsindex 689804e66..20201916d 100644--- a/cli/src/lib/remoteRelay.spec.ts+++ b/cli/src/lib/remoteRelay.spec.ts@@ -1,4 +1,4 @@-import { describe, expect, it, vi } from 'vitest'+import { afterEach, describe, expect, it, vi } from 'vitest'import { RemoteRelayPool } from './remoteRelay.js'/** Minimal fake of the private `Entry` shape `noteTerminalResponse`/`demoteP2p` operate on — exercised@@ -8,7 +8,7 @@ import { RemoteRelayPool } from './remoteRelay.js'function fakeEntry(overrides: Partial<Record<string, unknown>> = {}) {return {ws: { send: vi.fn() },- crypto: { wrapOutgoing: (frame: unknown) => frame },+ crypto: { wrapOutgoing: (frame: unknown) => frame, encryptTerminal: () => new Uint8Array([1, 2, 3]) },sink: { sendFrame: vi.fn(() => true), sendBinary: vi.fn(() => true) },onClosed: null,lingerTimer: null,@@ -301,6 +301,7 @@ function fakeP2p(overrides: Partial<Record<string, unknown>> = {}) {isReady: true,transport: 'direct',send: vi.fn(() => true),+ sendWithBackpressureRetry: vi.fn(async () => true),stop: vi.fn(async () => {}),...overrides,}@@ -450,3 +451,61 @@ describe('RemoteRelayPool TURN-to-direct upgrade', () => {expect(entry.upgradeTimer).toBeNull()})})++// Regression: a burst of large frames (a chunked upload's chunks, fired back-to-back) used to cross+// TERMINAL_P2P_MAX_BUFFERED_BYTES well before the real network drained the backlog, and a single+// resulting `send()` failure was misread as the whole p2p connection being dead — tearing it down for+// every stream, not just the one that hit backpressure. sendBinary must go through+// sendWithBackpressureRetry (which gives the channel one bounded chance to drain) rather than calling+// `send` directly, and must only fall back to relay + demote once that retry itself fails.+describe('RemoteRelayPool sendBinary backpressure handling', () => {+ const pool = new RemoteRelayPool(+ { accessToken: async () => 'unused' } as never,+ 'ws://unused',+ { pub: new Uint8Array(), priv: new Uint8Array() } as never,+ { pin: () => {}, get: () => null } as never,+ ) as unknown as {+ sessionFor: (machineId: string, entry: ReturnType<typeof fakeEntry>) => { sendBinary: (clear: { streamId: string }) => Promise<void> }+ demoteP2p: (machineId: string, entry: ReturnType<typeof fakeEntry>, reason: string) => void+ }++ // demoteP2p's own real behavior (resync-per-stream over ws, scheduleP2pRetry's real setTimeout) is+ // covered by the dedicated describe blocks above/below — stubbed out here to a no-op so these tests+ // observe only sendBinary's own routing decision, not demoteP2p's side effects or timers leaking+ // between tests.+ afterEach(() => { vi.restoreAllMocks() })++ it('sends over p2p via sendWithBackpressureRetry, not send, and never touches ws or demotes on success', async () => {+ const p2p = fakeP2p()+ const entry = fakeEntry({ p2p, p2pStreams: new Set(['stream-1']) })+ vi.spyOn(pool, 'demoteP2p').mockImplementation(() => {})+ await pool.sessionFor('machine-1', entry).sendBinary({ streamId: 'stream-1' })++ expect(p2p.sendWithBackpressureRetry).toHaveBeenCalledTimes(1)+ expect(p2p.send).not.toHaveBeenCalled()+ expect(entry.ws.send).not.toHaveBeenCalled()+ expect(pool.demoteP2p).not.toHaveBeenCalled()+ })++ it('falls back to ws and demotes only once sendWithBackpressureRetry itself fails', async () => {+ const p2p = fakeP2p({ sendWithBackpressureRetry: vi.fn(async () => false) })+ const entry = fakeEntry({ p2p, p2pStreams: new Set(['stream-1']) })+ const demoteP2p = vi.spyOn(pool, 'demoteP2p').mockImplementation(() => {})+ await pool.sessionFor('machine-1', entry).sendBinary({ streamId: 'stream-1' })++ expect(p2p.sendWithBackpressureRetry).toHaveBeenCalledTimes(1)+ expect(entry.ws.send).toHaveBeenCalledTimes(1) // the chunk still gets there, just over ws+ expect(demoteP2p).toHaveBeenCalledWith('machine-1', entry, 'send_failed')+ })++ it('goes straight to ws with no demotion for a stream that was never on p2p', async () => {+ const p2p = fakeP2p()+ const entry = fakeEntry({ p2p, p2pStreams: new Set() }) // this stream never made it onto p2pStreams+ const demoteP2p = vi.spyOn(pool, 'demoteP2p').mockImplementation(() => {})+ await pool.sessionFor('machine-1', entry).sendBinary({ streamId: 'stream-1' })++ expect(p2p.sendWithBackpressureRetry).not.toHaveBeenCalled()+ expect(entry.ws.send).toHaveBeenCalledTimes(1)+ expect(demoteP2p).not.toHaveBeenCalled()+ })+})diff --git a/cli/src/lib/remoteRelay.ts b/cli/src/lib/remoteRelay.tsindex 652c6fba1..0f2769e20 100644--- a/cli/src/lib/remoteRelay.ts+++ b/cli/src/lib/remoteRelay.ts@@ -504,8 +504,21 @@ export class RemoteRelayPool {sendBinary: async (clear) => {const sealed = entry.crypto.encryptTerminal(clear)if (!sealed) return- if (entry.p2pStreams.has(clear.streamId) && entry.p2p?.send(Buffer.from(sealed))) return- const p2pFailed = entry.p2pStreams.has(clear.streamId)+ const wantsP2p = entry.p2pStreams.has(clear.streamId)+ // A burst of large frames (a chunked upload's chunks, fired back-to-back) can push the data+ // channel's send buffer over TERMINAL_P2P_MAX_BUFFERED_BYTES well before the real network has+ // drained it — sendWithBackpressureRetry gives that ONE bounded chance to clear before this+ // falls through to relay-and-demote, instead of misreading a busy channel as a dead one. A+ // plain successful send resolves in microseconds, so timing the call is enough to tell "it had+ // to wait" apart from the ordinary case without threading a reporting hook through the p2p class.+ const startedAt = Date.now()+ const p2pSent = wantsP2p && await entry.p2p?.sendWithBackpressureRetry(Buffer.from(sealed))+ const waited = Date.now() - startedAt >= 50+ if (waited) {+ console.log(`[p2p] backpressure · machine=${sid(machineId)} drained=${p2pSent ? 'yes' : 'no'} after=${Date.now() - startedAt}ms`)+ }+ if (p2pSent) return+ const p2pFailed = wantsP2ptry { entry.ws.send(sealed, { binary: true }) } catch { /* closed — onClosed will fire */ }if (p2pFailed) this.demoteP2p(machineId, entry, 'send_failed')},diff --git a/cli/src/lib/terminalP2p.spec.ts b/cli/src/lib/terminalP2p.spec.tsindex ad4024747..8b2ccd254 100644--- a/cli/src/lib/terminalP2p.spec.ts+++ b/cli/src/lib/terminalP2p.spec.ts@@ -7,10 +7,26 @@ import {TERMINAL_P2P_UP_TYPES,TerminalP2pInitiator,TerminalP2pResponderPool,+ waitForBufferedAmountLow,type TerminalP2pSignal,} from './terminalP2p.js'import type { StunSelector } from './stunSelect.js'+/** Structurally shaped like the slice of werift's `RTCDataChannel` this function actually reads —+ * no real WebRTC negotiation needed, same reasoning as `isRelayedPair`'s plain-string fixtures. */+function fakeChannel(initial: {+ readyState?: 'open' | 'closed' | 'connecting' | 'closing'+ bufferedAmount?: number+ asPromise?: () => Promise<unknown[]>+}) {+ return {+ readyState: initial.readyState ?? 'open',+ bufferedAmount: initial.bufferedAmount ?? 0,+ bufferedAmountLow: { asPromise: initial.asPromise ?? (() => new Promise<unknown[]>(() => {})) },+ }+}+type FakeChannel = ReturnType<typeof fakeChannel>+describe('terminal WebRTC data channel', () => {it('negotiates locally and carries ordered text and binary frames in both directions', async () => {let resolveResponderData!: (value: string) => void@@ -250,6 +266,43 @@ describe('terminal WebRTC data channel', () => {expect(isRelayedPair('candidate:1 1 udp 1 10.0.0.1 1 typ host raddr relay.example')).toBe(false)})+ // Regression: a burst of sends (a chunked upload's chunks, fired back-to-back) can cross+ // TERMINAL_P2P_MAX_BUFFERED_BYTES well before the real network drains the backlog — this is the+ // wait this puts to use instead of every send() in that window being mistaken for a dead channel.+ describe('waitForBufferedAmountLow', () => {+ const wait = (channel: FakeChannel, timeoutMs = 1_000) =>+ waitForBufferedAmountLow(channel as unknown as Parameters<typeof waitForBufferedAmountLow>[0], timeoutMs)++ it('resolves immediately when the buffer is already under the ceiling', async () => {+ const channel = fakeChannel({ bufferedAmount: 0 })+ await expect(wait(channel)).resolves.toBe(true)+ })++ it('fails fast with no wait when the channel is not open', async () => {+ const asPromise = vi.fn(() => new Promise<unknown[]>(() => {}))+ const channel = fakeChannel({ readyState: 'closed', bufferedAmount: 5_000_000, asPromise })+ await expect(wait(channel)).resolves.toBe(false)+ expect(asPromise).not.toHaveBeenCalled() // a dead channel is never worth waiting on+ })++ it('waits for the drain event, then re-checks the buffer before resolving true', async () => {+ const channel = fakeChannel({ bufferedAmount: 5_000_000, asPromise: () => Promise.resolve([]) })+ const result = wait(channel)+ channel.bufferedAmount = 0 // the buffer clears right as the event fires+ await expect(result).resolves.toBe(true)+ })++ it('resolves false if the buffer is still over the ceiling once the event fires', async () => {+ const channel = fakeChannel({ bufferedAmount: 5_000_000, asPromise: () => Promise.resolve([]) })+ await expect(wait(channel)).resolves.toBe(false) // bufferedAmount never actually dropped+ })++ it('resolves false when the wait itself times out', async () => {+ const channel = fakeChannel({ bufferedAmount: 5_000_000, asPromise: () => Promise.reject(new Error('timeout')) })+ await expect(wait(channel)).resolves.toBe(false)+ })+ })+// Regression: a JSON type these two allowlists don't know about is silently dropped the instant a// pane's connection is on a direct p2p data channel (backendSocket.ts's `handleP2pData` / remoteRelay.ts's// p2p-data handler) — no error, no log, indistinguishable from the wire itself hanging. That's exactlydiff --git a/cli/src/lib/terminalP2p.ts b/cli/src/lib/terminalP2p.tsindex 40856b1dd..683c70eb4 100644--- a/cli/src/lib/terminalP2p.ts+++ b/cli/src/lib/terminalP2p.ts@@ -189,6 +189,27 @@ function channelCanSend(channel: RTCDataChannel | null): channel is RTCDataChann&& channel.bufferedAmount < TERMINAL_P2P_MAX_BUFFERED_BYTES}+/**+ * Waits for `channel`'s send buffer to drop back under the ceiling, using the+ * `bufferedAmountLowThreshold`/`bufferedAmountLow` plumbing this file already configures+ * (`wireChannel`/`wireResponderEntry` below) but never previously consulted — a burst of sends (a+ * chunked upload's chunks, fired back-to-back) can cross `TERMINAL_P2P_MAX_BUFFERED_BYTES` well+ * before the real network has drained the backlog, and without this wait every `send()` call in that+ * window was mistaken for the channel being dead outright. Resolves `false` on a genuine channel+ * closure or if the buffer hasn't cleared within `timeoutMs` — either way, the caller's existing+ * fall-back-to-relay-and-demote behavior is exactly what should happen next.+ */+export async function waitForBufferedAmountLow(channel: RTCDataChannel, timeoutMs: number): Promise<boolean> {+ if (channel.readyState !== 'open') return false+ if (channel.bufferedAmount < TERMINAL_P2P_MAX_BUFFERED_BYTES) return true+ try {+ await channel.bufferedAmountLow.asPromise(timeoutMs)+ } catch {+ return false // timed out, or the channel errored while we waited+ }+ return channel.readyState === 'open' && channel.bufferedAmount < TERMINAL_P2P_MAX_BUFFERED_BYTES+}+/** Source side: owns the offerer for one pooled remote-machine relay connection. */export class TerminalP2pInitiator {readonly sessionId = randomUUID()@@ -343,6 +364,23 @@ export class TerminalP2pInitiator {}}+ /**+ * Same contract as `send()`, but a failure caused purely by backpressure (the channel is still+ * `open`, just over `TERMINAL_P2P_MAX_BUFFERED_BYTES` right now) gets ONE bounded chance to drain+ * before being reported as failure — see `waitForBufferedAmountLow`'s own comment for why this+ * exists. A `send()` failure for any other reason (channel not open/ready, or `channel.send()`+ * itself threw, which schedules `fail()`) is NOT retried: re-reading `channel.readyState` fresh+ * rather than trusting `this.isReady` is what tells the two apart, since a queued `fail()` from a+ * hard send error hasn't necessarily flipped `this.ready` by the time this runs.+ */+ async sendWithBackpressureRetry(data: TerminalP2pData, drainTimeoutMs = 5_000): Promise<boolean> {+ if (this.send(data)) return true+ const channel = this.channel+ if (!channel || channel.readyState !== 'open') return false+ if (!(await waitForBufferedAmountLow(channel, drainTimeoutMs))) return false+ return this.send(data)+ }+waitUntilReady(timeoutMs: number): Promise<boolean> {if (this.isReady) return Promise.resolve(true)if (this.finished || timeoutMs <= 0) return Promise.resolve(false)