Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

fix(p2p): a TURN TLS reset must not crash the gateway (#1844) (#1845)

Clovera6599c125d91
diff --git a/cli/src/lib/terminalP2p.ts b/cli/src/lib/terminalP2p.ts
index 19beb34d9..da7f5f55b 100644
--- a/cli/src/lib/terminalP2p.ts
+++ b/cli/src/lib/terminalP2p.ts
@@ -3,6 +3,7 @@ import { BlockList, isIP } from 'node:net'
import type { RTCPeerConnection, RTCDataChannel, RTCIceCandidateInit } from 'werift'
import { env } from '../config/env.js'
import { pickTurnUrl, selectStunUrls, type StunSelector } from './stunSelect.js'
+import { armTurnTls } from './turnTlsGuard.js'
export const TERMINAL_P2P_PROTOCOL_VERSION = 1
export const TERMINAL_P2P_CHANNEL = 'terminal-v1'
@@ -369,7 +370,11 @@ export class TerminalP2pInitiator {
// strand it: nothing holds a reference any more, so its UDP sockets would never be closed.
if (this.finished) return
this.step('stun-raced')
- const pc = new RTCPeerConnection(peerConfig(stunUrls, turnChoice(this.deps.policy.turn, udpReachable), this.deps.relayOnly))
+ const turn = turnChoice(this.deps.policy.turn, udpReachable)
+ // Before gather dials. A TLS reset on this socket used to reject an unhandled promise
+ // and exit the gateway (armTurnTls).
+ armTurnTls(turn)
+ const pc = new RTCPeerConnection(peerConfig(stunUrls, turn, this.deps.relayOnly))
const channel = pc.createDataChannel(TERMINAL_P2P_CHANNEL, { ordered: true })
this.pc = pc
this.channel = channel
@@ -752,6 +757,8 @@ export class TerminalP2pResponderPool {
const { RTCPeerConnection } = await import('werift')
if (!stillCurrent()) return null
const turn = turnChoice(readTurn(payload.turn), selection.udpReachable)
+ // Same guard as the offerer: this is the side that dials turns:443 when UDP did not answer.
+ armTurnTls(turn)
const pc = new RTCPeerConnection(peerConfig(selection.urls, turn))
const entry: ResponderEntry = {
sessionId: payload.sessionId,
diff --git a/cli/src/lib/terminalP2pTurnTls.spec.ts b/cli/src/lib/terminalP2pTurnTls.spec.ts
new file mode 100644
index 000000000..6fe8ea5e7
--- /dev/null
+++ b/cli/src/lib/terminalP2pTurnTls.spec.ts
@@ -0,0 +1,192 @@
+import { execFileSync } from 'node:child_process'
+import { mkdtempSync, readFileSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { connect, createServer, type Server, type TLSSocket } from 'node:tls'
+import { afterEach, describe, expect, it, vi } from 'vitest'
+import { TerminalP2pInitiator, TerminalP2pResponderPool, type TerminalP2pSignal } from './terminalP2p.js'
+import { armTurnTls } from './turnTlsGuard.js'
+
+/**
+ * A TLS server that accepts TCP and destroys the socket before the handshake finishes.
+ * Node reports that to the client as ECONNRESET
+ * "Client network socket disconnected before secure TLS connection was established",
+ * with the host and port of the dial — the same error that ends the gateway.
+ */
+function testMaterial(): { key: string; cert: string } {
+ const dir = mkdtempSync(join(tmpdir(), 'turn-tls-reset-'))
+ const keyPath = join(dir, 'key.pem')
+ const certPath = join(dir, 'cert.pem')
+ execFileSync('openssl', [
+ 'req', '-x509', '-newkey', 'ec', '-pkeyopt', 'ec_paramgen_curve:prime256v1',
+ '-keyout', keyPath, '-out', certPath, '-days', '1', '-nodes', '-subj', '/CN=localhost',
+ ], { stdio: 'ignore' })
+ return { key: readFileSync(keyPath, 'utf8'), cert: readFileSync(certPath, 'utf8') }
+}
+
+function resettingTlsServer(): Promise<{ port: number; connections: () => number; close: () => Promise<void> }> {
+ const material = testMaterial()
+ let connections = 0
+ const server: Server = createServer(
+ material,
+ (_socket: TLSSocket) => { /* handshake completed; the reset below should prevent this */ },
+ )
+ server.on('connection', (socket) => {
+ connections += 1
+ socket.destroy()
+ })
+ return new Promise((resolve, reject) => {
+ server.once('error', reject)
+ server.listen(0, '127.0.0.1', () => {
+ const address = server.address()
+ if (!address || typeof address === 'string') { reject(new Error('expected a tcp port')); return }
+ resolve({
+ port: address.port,
+ connections: () => connections,
+ close: () => new Promise((done) => { server.close(() => done()) }),
+ })
+ })
+ })
+}
+
+function rejectionText(reason: unknown): string {
+ if (reason instanceof Error) return `${reason.message} ${(reason as NodeJS.ErrnoException).code ?? ''}`
+ return String(reason)
+}
+
+describe('TURN over TLS reset', () => {
+ const rejections: unknown[] = []
+ const onUnhandled = (reason: unknown): void => { rejections.push(reason) }
+
+ afterEach(() => {
+ process.off('unhandledRejection', onUnhandled)
+ rejections.length = 0
+ })
+
+ it('does not kill the process, and a direct pair still opens', async () => {
+ const turn = await resettingTlsServer()
+ process.on('unhandledRejection', onUnhandled)
+ const log = vi.spyOn(console, 'log').mockImplementation(() => {})
+ let initiator!: TerminalP2pInitiator
+ let responderData = ''
+ const responder = new TerminalP2pResponderPool({
+ // UDP did not answer, so the responder must pick the turns: url (the restricted-network case).
+ selectStunUrls: async () => ({ urls: [], udpReachable: false }),
+ sendSignal: (_connId, type, payload) => { void initiator.handleSignal(type, payload) },
+ onData: (_connId, data) => { responderData = typeof data === 'string' ? data : data.toString() },
+ })
+ initiator = new TerminalP2pInitiator({
+ policy: { enabled: true, protocolVersion: 1, stunUrls: [], openWaitMs: 1_500 },
+ selectStunUrls: async () => ({ urls: [], udpReachable: null }),
+ sendSignal: (type, payload) => {
+ const signal: TerminalP2pSignal = type === 'p2p_offer'
+ ? {
+ ...payload,
+ turn: {
+ urls: [
+ 'turn:127.0.0.1:3478?transport=udp',
+ `turns:127.0.0.1:${turn.port}?transport=tcp`,
+ ],
+ username: 'user',
+ credential: 'credential',
+ },
+ }
+ : payload
+ void responder.handleSignal('source-1', type, signal)
+ },
+ onData: () => { /* the responder is the side that dials TURN */ },
+ })
+
+ try {
+ initiator.start()
+ expect(await initiator.waitUntilReady(8_000)).toBe(true)
+ // The orphaned reconnect rejects on a later turn of the event loop than the handshake error.
+ await new Promise((resolve) => setTimeout(resolve, 500))
+ expect(rejections.map(rejectionText)).toEqual([])
+ expect(initiator.send('still-up')).toBe(true)
+ await new Promise((resolve) => setTimeout(resolve, 200))
+ expect(responderData).toBe('still-up')
+ // One dial. A reconnect loop would open a socket per failure.
+ expect(turn.connections()).toBe(1)
+ } finally {
+ log.mockRestore()
+ await initiator?.stop('test_complete', false)
+ await responder.stop()
+ await turn.close()
+ }
+ }, 15_000)
+
+ it('fails a relay-only negotiation instead of rejecting out of the process', async () => {
+ const turn = await resettingTlsServer()
+ process.on('unhandledRejection', onUnhandled)
+ const states: string[] = []
+ const signals: string[] = []
+ const initiator = new TerminalP2pInitiator({
+ policy: {
+ enabled: true,
+ protocolVersion: 1,
+ stunUrls: [],
+ openWaitMs: 1_500,
+ turn: {
+ urls: [
+ 'turn:127.0.0.1:3478?transport=udp',
+ `turns:127.0.0.1:${turn.port}?transport=tcp`,
+ ],
+ username: 'user',
+ credential: 'credential',
+ },
+ },
+ relayOnly: true,
+ selectStunUrls: async () => ({ urls: [], udpReachable: false }),
+ sendSignal: (type) => { signals.push(type) },
+ onData: () => { /* no peer answers; the ws relay would keep the terminal */ },
+ onState: (state) => { states.push(state) },
+ })
+
+ try {
+ initiator.start()
+ const started = Date.now()
+ while (rejections.length === 0 && signals.length === 0 && Date.now() - started < 8_000) {
+ await new Promise((resolve) => setTimeout(resolve, 20))
+ }
+ await new Promise((resolve) => setTimeout(resolve, 500))
+ expect(rejections.map(rejectionText)).toEqual([])
+ // Gathering survived the reset, so an offer went out with no relay candidate to nominate.
+ // Nothing answers it; the existing negotiation timeout is what returns the terminal to the relay.
+ expect(signals).toContain('p2p_offer')
+ expect(states[0]).toBe('connecting')
+ expect(turn.connections()).toBe(1)
+ } finally {
+ await initiator.stop('test_complete', false)
+ await turn.close()
+ }
+ }, 15_000)
+
+ it('still reports end once the TURN TLS handshake has finished', async () => {
+ const material = testMaterial()
+ let port = 0
+ const server = createServer({ key: material.key, cert: material.cert }, (socket) => { socket.end() })
+ await new Promise<void>((resolve, reject) => {
+ server.once('error', reject)
+ server.listen(0, '127.0.0.1', () => {
+ const address = server.address()
+ if (!address || typeof address === 'string') { reject(new Error('expected a tcp port')); return }
+ port = address.port
+ resolve()
+ })
+ })
+ armTurnTls({ url: `turns:127.0.0.1:${port}?transport=tcp` })
+ const socket = connect({ host: '127.0.0.1', port, rejectUnauthorized: false })
+ const ended = new Promise<string>((resolve) => { socket.on('end', () => resolve('ended')) })
+ try {
+ await new Promise<void>((resolve, reject) => {
+ socket.once('secureConnect', () => resolve())
+ socket.once('error', reject)
+ })
+ expect(await ended).toBe('ended')
+ } finally {
+ socket.destroy()
+ await new Promise<void>((resolve) => { server.close(() => resolve()) })
+ }
+ })
+})
diff --git a/cli/src/lib/turnTlsGuard.ts b/cli/src/lib/turnTlsGuard.ts
new file mode 100644
index 000000000..90a2aa6ac
--- /dev/null
+++ b/cli/src/lib/turnTlsGuard.ts
@@ -0,0 +1,120 @@
+import tls, { type TLSSocket } from 'node:tls'
+
+/**
+ * A TURN-over-TLS failure must not take down the gateway.
+ *
+ * werift 0.24.4 dials `turns:` with `tls.connect` and then, in `StreamTransport.connect`
+ * (`werift/lib/common/src/transport.js`), does `client.on("end", () => this.connect())`.
+ * A peer that accepts TCP and drops it before the handshake emits `end` and then
+ * `error`: Node's `ECONNRESET`, "Client network socket disconnected before secure TLS
+ * connection was established", with `host` and `port`. The `end` handler replaces
+ * `this.connecting` and dials again. The gatherer is awaiting the first promise, and
+ * werift's `ice.ts` catches that rejection (`query TURN server`). Nothing awaits the
+ * replacement, so the rejection is unhandled. Node reports it through
+ * `node:internal/process/promises` `triggerUncaughtException` and the gateway process
+ * exits. harnessd restarts it and, after repeated crashes, parks it. The encrypted
+ * relay was never what failed.
+ *
+ * `pickTurnUrl` selects this dial when UDP did not answer. On the public list that
+ * url is `turns:` port 443.
+ *
+ * The guard wraps `end` on that socket until `secureConnect`. The handshake error
+ * stays on the promise werift already catches, gathering finishes with no relay
+ * candidate, and the session stays on a direct pair when one exists. Otherwise the
+ * negotiation fails and the terminal stays on the encrypted WebSocket relay. An
+ * `end` after the handshake still reconnects.
+ *
+ * Wrapping the socket, rather than replacing `tls.connect`, is what reaches the
+ * listener: werift calls whichever `tls.connect` it closed over. Patching werift is
+ * not required. The library already catches the failed dial. The crash is only the
+ * reconnect that this premature `end` starts.
+ */
+const endpoints = new Set<string>()
+let installed = false
+
+export function armTurnTls(turn: { url: string } | undefined): void {
+ if (!turn) return
+ const endpoint = turnsEndpoint(turn.url)
+ if (!endpoint) return
+ endpoints.add(endpointKey(endpoint.host, endpoint.port))
+ install()
+}
+
+type SocketOn = (this: TLSSocket, event: string | symbol, listener: (...args: unknown[]) => void, ...rest: unknown[]) => TLSSocket
+
+function install(): void {
+ if (installed) return
+ installed = true
+ const originalOn = tls.TLSSocket.prototype.on as unknown as SocketOn
+ const guardedOn: SocketOn = function guardedTurnTlsOn(event, listener, ...rest) {
+ // Node's own onConnectEnd is registered before the connect options are visible.
+ // Leaving it in place is what turns the close into the error werift already
+ // catches. The listener added after that is the reconnect. `encrypted` cannot
+ // tell these apart: it is true on every TLSSocket from the moment it exists.
+ // getCipher() stays empty until the handshake finishes.
+ if (event === 'end' && listener.name !== 'onConnectEnd' && isGuardedDial(this)) {
+ let secure = this.getCipher() != null
+ if (!secure) this.once('secureConnect', () => { secure = true })
+ const afterHandshake = (...args: unknown[]): void => {
+ if (!secure) return
+ listener.apply(this, args)
+ }
+ return originalOn.call(this, event, afterHandshake, ...rest)
+ }
+ return originalOn.call(this, event, listener, ...rest)
+ }
+ tls.TLSSocket.prototype.on = guardedOn as typeof tls.TLSSocket.prototype.on
+ tls.TLSSocket.prototype.addListener = guardedOn as typeof tls.TLSSocket.prototype.addListener
+}
+
+function isGuardedDial(socket: TLSSocket): boolean {
+ const target = dialTarget(socket)
+ return !!target && endpoints.has(endpointKey(target.host, target.port))
+}
+
+/** Node stores the `tls.connect` options on the socket under a symbol. That is the dial werift made. */
+function dialTarget(socket: TLSSocket): { host: string; port: number } | null {
+ for (const key of Object.getOwnPropertySymbols(socket)) {
+ const value = (socket as unknown as Record<symbol, unknown>)[key]
+ if (!value || typeof value !== 'object') continue
+ const host = (value as { host?: unknown }).host
+ const port = (value as { port?: unknown }).port
+ if (typeof host === 'string' && host && typeof port === 'number' && Number.isInteger(port)) {
+ return { host, port }
+ }
+ }
+ return null
+}
+
+function endpointKey(host: string, port: number): string {
+ return `${host.toLowerCase()}:${port}`
+}
+
+/**
+ * Host and port werift passes to `tls.connect`. A `turns:` url with no port is 5349
+ * (RFC 7065), which is werift's fallback too, so the guard matches the dial.
+ */
+function turnsEndpoint(url: string): { host: string; port: number } | null {
+ const match = /^turns:(?:\/\/)?([^?]+)/i.exec(url.trim())
+ if (!match) return null
+ const authority = match[1]!
+ if (authority.startsWith('[')) {
+ const end = authority.indexOf(']')
+ if (end < 0) return null
+ const host = authority.slice(1, end)
+ return host ? { host, port: portAfter(authority.slice(end + 1)) } : null
+ }
+ const firstColon = authority.indexOf(':')
+ const lastColon = authority.lastIndexOf(':')
+ if (firstColon === -1 || firstColon !== lastColon) return authority ? { host: authority, port: 5349 } : null
+ const host = authority.slice(0, firstColon)
+ if (!host) return null
+ return { host, port: portAfter(authority.slice(firstColon)) }
+}
+
+function portAfter(rest: string): number {
+ const text = rest.startsWith(':') ? rest.slice(1) : rest
+ if (!text) return 5349
+ const port = Number.parseInt(text, 10)
+ return Number.isInteger(port) && port > 0 && port < 65536 ? port : 5349
+}