Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

Merge origin/dev into fix/hcc-resource-rollout-bootstrap

Alfredo Lopez7183ff41ef7e
diff --cc scripts/tests/wrc-egress-dns-proxy.test.cjs
index 9d1ce73e0,01374c471..198238642
--- a/scripts/tests/wrc-egress-dns-proxy.test.cjs
+++ b/scripts/tests/wrc-egress-dns-proxy.test.cjs
@@@ -138,218 -109,6 +138,218 @@@ async function fixture(t, configuredTar
return { proxy, ports, control }
}
+test(
+ 'ephemeral DNS binding recovers from a real UDP collision and preserves runtime faults',
+ { timeout: 5000 },
+ async t => {
+ const occupied = await occupiedDnsTransport(t, 'UDP')
+ const occupiedPort = occupied.address().port
+ const dnsListeners = []
+ const dnsDatagrams = []
+ const nativeCreateSocket = dgram.createSocket
+ t.mock.method(dgram, 'createSocket', function (...args) {
+ const socket = nativeCreateSocket.apply(this, args)
+ if (dnsDatagrams.length < 2) dnsDatagrams.push(socket)
+ return socket
+ })
+ const nativeListen = net.Server.prototype.listen
+ // Control only the first OS candidate, not the sockets or DNS exchange.
+ // The already-bound UDP socket makes the opposite-transport collision real.
+ t.mock.method(net.Server.prototype, 'listen', function (port, ...args) {
+ if (port === 0 && dnsListeners.length < 2) {
+ dnsListeners.push(this)
+ if (dnsListeners.length === 1) port = occupiedPort
+ }
+ return nativeListen.call(this, port, ...args)
+ })
+ const proxy = createDnsProxy({
+ targets,
+ upstream: '127.0.0.1',
+ listenHost: '127.0.0.1',
+ dnsPort: 0,
+ controlPort: 0,
+ ttlSeconds: 10,
+ })
+ t.after(() => proxy.close())
+ const ports = await proxy.start()
+ assert.equal(dnsListeners.length, 2)
+ assert.equal(dnsListeners[0].listening, false, 'the failed TCP reservation is released')
+ assert.equal(dnsListeners[0].address(), null)
+ assert.throws(() => dnsDatagrams[0].address(), { code: 'ERR_SOCKET_DGRAM_NOT_RUNNING' })
+ assert.equal(dnsListeners[1].address().port, ports.dnsPort)
+ assert.equal(dnsDatagrams[1].address().port, ports.dnsPort)
+ assert.notEqual(ports.dnsPort, occupiedPort)
+ assert.equal(proxy.snapshot().fault, null, 'a recovered bind collision is not a runtime fault')
+ const client = await udpClient(t)
+ const wire = query(targets[0].fqdn, 201)
+ const expected = dnsResponse(wire, question(wire), 0, targets[0].ip, 10)
+ assert.deepEqual(await exchangeUdp(client, ports.dnsPort, wire), expected)
+ assert.deepEqual(await exchangeTcp(ports.dnsPort, wire), expected)
+ assert.equal(proxy.snapshot().counts.ok, 2)
+ // A post-start socket error still invalidates fixture evidence.
+ for (const listener of [dnsListeners[1], dnsDatagrams[1]]) {
+ listener.emit('error', new Error('test runtime DNS socket error'))
+ assert.equal(proxy.snapshot().fault, 'DNS_SOCKET_FAILED')
+ assert.equal(proxy.snapshot().counts.ok, 2)
+ }
+ }
+)
+
+test(
- 'ephemeral opposite-transport collisions stop after eight attempts and release every pair',
++ 'ephemeral opposite-transport collisions stop after twenty attempts and release every pair',
+ { timeout: 5000 },
+ async t => {
+ const occupied = await occupiedDnsTransport(t, 'UDP')
+ const occupiedPort = occupied.address().port
+ const attempts = []
+ await assert.rejects(
+ listenDnsPair(
+ () => {
+ const udp = dgram.createSocket('udp4')
+ const tcp = net.createServer()
+ const nativeListen = tcp.listen.bind(tcp)
+ tcp.listen = (port, ...args) => nativeListen(port === 0 ? occupiedPort : port, ...args)
+ attempts.push({ udp, tcp })
+ // Independent cleanup also keeps a broken cleanup implementation
+ // from leaving this negative test's real sockets alive.
+ t.after(async () => {
+ await new Promise(resolve => tcp.close(() => resolve()))
+ try {
+ udp.close()
+ } catch (error) {
+ assert.equal(error.code, 'ERR_SOCKET_DGRAM_NOT_RUNNING')
+ }
+ })
+ return { udp, tcp }
+ },
+ 0,
+ '127.0.0.1'
+ ),
+ { code: 'EADDRINUSE', port: occupiedPort }
+ )
- assert.equal(attempts.length, 8)
++ assert.equal(attempts.length, 20)
+ for (const { udp, tcp } of attempts) {
+ assert.equal(tcp.listening, false)
+ assert.equal(tcp.address(), null)
+ assert.throws(() => udp.address(), { code: 'ERR_SOCKET_DGRAM_NOT_RUNNING' })
+ }
+ assert.equal(occupied.address().port, occupiedPort)
+ await tcpListener(t, occupiedPort)
+ }
+)
+
+test(
+ 'failed DNS startup releases TCP clients accepted before an opposite-transport collision',
+ { timeout: 5000 },
+ async t => {
+ const occupied = await occupiedDnsTransport(t, 'UDP')
+ const occupiedPort = occupied.address().port
+ let listeners
+ let client
+ let accepted
+ t.after(async () => {
+ client?.destroy()
+ accepted?.destroy()
+ if (listeners) await closeDnsListeners(listeners)
+ })
+ await assert.rejects(
+ listenDnsPair(
+ () => {
+ const udp = dgram.createSocket('udp4')
+ const tcp = net.createServer(socket => {
+ accepted = socket
+ })
+ const nativeBind = udp.bind.bind(udp)
+ // Delay only the bind until a real TCP client has been accepted;
+ // the subsequent UDP bind still fails against the occupied socket.
+ udp.bind = (port, host) => {
+ const connected = once(tcp, 'connection')
+ client = net.createConnection({ port, host })
+ client.on('error', error => assert.equal(error.code, 'ECONNRESET'))
+ connected.then(() => nativeBind(port, host))
+ return udp
+ }
+ listeners = { udp, tcp }
+ return listeners
+ },
+ occupiedPort,
+ '127.0.0.1'
+ ),
+ { code: 'EADDRINUSE', port: occupiedPort }
+ )
+ assert.equal(accepted.destroyed, true)
+ assert.equal(listeners.tcp.address(), null)
+ assert.throws(() => listeners.udp.address(), { code: 'ERR_SOCKET_DGRAM_NOT_RUNNING' })
+ assert.equal(occupied.address().port, occupiedPort)
+ }
+)
+
+test(
+ 'an ephemeral TCP allocation failure does not retry as an opposite-transport collision',
+ { timeout: 5000 },
+ async t => {
+ const occupied = await tcpListener(t)
+ const occupiedPort = occupied.address().port
+ const attempts = []
+ await assert.rejects(
+ listenDnsPair(
+ () => {
+ const udp = dgram.createSocket('udp4')
+ const tcp = net.createServer()
+ const nativeListen = tcp.listen.bind(tcp)
+ tcp.listen = (port, ...args) => nativeListen(port === 0 ? occupiedPort : port, ...args)
+ attempts.push({ udp, tcp })
+ return { udp, tcp }
+ },
+ 0,
+ '127.0.0.1'
+ ),
+ { code: 'EADDRINUSE', port: occupiedPort }
+ )
+ assert.equal(attempts.length, 1)
+ assert.equal(attempts[0].tcp.address(), null)
+ assert.throws(() => attempts[0].udp.address(), { code: 'ERR_SOCKET_DGRAM_NOT_RUNNING' })
+ assert.equal(occupied.address().port, occupiedPort)
+ }
+)
+
+for (const transport of ['UDP', 'TCP']) {
+ test(
+ `a fixed DNS port occupied by ${transport} fails once without changing the requested port`,
+ { timeout: 5000 },
+ async t => {
+ const occupied = await occupiedDnsTransport(t, transport)
+ const occupiedPort = occupied.address().port
+ const nativeListen = net.Server.prototype.listen
+ let attempts = 0
+ t.mock.method(net.Server.prototype, 'listen', function (port, ...args) {
+ if (port === occupiedPort) attempts++
+ return nativeListen.call(this, port, ...args)
+ })
+ const proxy = createDnsProxy({
+ targets,
+ upstream: '127.0.0.1',
+ listenHost: '127.0.0.1',
+ dnsPort: occupiedPort,
+ controlPort: 0,
+ })
+ t.after(() => proxy.close())
+ await assert.rejects(proxy.start(), { code: 'EADDRINUSE', port: occupiedPort })
+ assert.equal(attempts, 1)
+ assert.equal(proxy.snapshot().counts.ok, 0)
+ assert.equal(occupied.address().port, occupiedPort)
+ if (transport === 'UDP') await tcpListener(t, occupiedPort)
+ else {
+ const udp = dgram.createSocket('udp4')
+ udp.bind(occupiedPort, '127.0.0.1')
+ await once(udp, 'listening', { signal: AbortSignal.timeout(2000) })
+ t.after(() => new Promise(resolve => udp.close(resolve)))
+ }
+ await assert.rejects(proxy.start(), /cannot be started twice/)
+ }
+ )
+}
+
test('target configuration is explicit, bounded, unambiguous, and safe for arbitrary names', () => {
const valid = { targets, upstream: '127.0.0.1' }
assert.throws(() => readConfig({ TARGET_DNS: 'legacy.example.com', UPSTREAM_DNS: '127.0.0.1' }))
@@@ -651,3 -410,53 +651,59 @@@ test
await proxy.close()
}
)
+
-// A UDP socket stand-in whose bind reports EADDRINUSE for the first `collisions` calls.
-function collidingUdp(collisions) {
++// A synthetic UDP listener makes the retry budget deterministic; TCP still
++// reserves real loopback ports. Each failed attempt gets a fresh listener.
++function collidingDnsListeners(collisions) {
+ const { EventEmitter } = require('node:events')
- const udp = new EventEmitter()
- udp.bound = []
- udp.bind = port => {
- udp.bound.push(port)
- setImmediate(() => {
- if (udp.bound.length <= collisions) {
- udp.emit('error', Object.assign(new Error('bind EADDRINUSE'), { code: 'EADDRINUSE' }))
- } else {
- udp.emit('listening')
- }
- })
++ const bound = []
++ const listeners = []
++ const create = () => {
++ const udp = new EventEmitter()
++ udp.bind = port => {
++ bound.push(port)
++ setImmediate(() => {
++ if (bound.length <= collisions) {
++ udp.emit('error', Object.assign(new Error('bind EADDRINUSE'), { code: 'EADDRINUSE' }))
++ } else {
++ udp.emit('listening')
++ }
++ })
++ }
++ udp.close = callback => setImmediate(callback)
++ const pair = { udp, tcp: net.createServer() }
++ listeners.push(pair)
++ return pair
+ }
- return udp
++ return { create, bound, listeners }
+ }
+
+ test('an ephemeral DNS port pair is reallocated when UDP finds the TCP port taken', async t => {
- const udp = collidingUdp(2)
- const tcp = net.createServer()
- t.after(() => new Promise(resolve => (tcp.listening ? tcp.close(resolve) : resolve())))
- const port = await listenDnsPair(udp, tcp, 0, '127.0.0.1')
- assert.equal(udp.bound.length, 3, 'two collisions, then a pair that binds')
- assert.equal(tcp.listening, true)
- assert.equal(tcp.address().port, port)
- assert.equal(udp.bound[2], port, 'UDP joins the port TCP chose')
++ const fixture = collidingDnsListeners(2)
++ const pair = await listenDnsPair(fixture.create, 0, '127.0.0.1')
++ t.after(() => closeDnsListeners(pair))
++ const port = pair.tcp.address().port
++ assert.equal(fixture.bound.length, 3, 'two collisions, then a pair that binds')
++ assert.equal(pair.tcp.listening, true)
++ assert.equal(fixture.bound[2], port, 'UDP joins the port TCP chose')
++ assert.ok(fixture.listeners.slice(0, -1).every(({ tcp }) => !tcp.listening))
+ })
+
+ test('an explicit or exhausted DNS port pair rethrows EADDRINUSE and releases TCP', async () => {
+ const blocker = net.createServer()
+ blocker.listen(0, '127.0.0.1')
+ await once(blocker, 'listening')
- const fixedUdp = collidingUdp(1)
- const fixedTcp = net.createServer()
+ const free = blocker.address().port
+ await new Promise(resolve => blocker.close(resolve))
- await assert.rejects(listenDnsPair(fixedUdp, fixedTcp, free, '127.0.0.1'), { code: 'EADDRINUSE' })
- assert.equal(fixedUdp.bound.length, 1, 'an explicit port gets one attempt')
- assert.equal(fixedTcp.listening, false)
++ const fixed = collidingDnsListeners(1)
++ await assert.rejects(listenDnsPair(fixed.create, free, '127.0.0.1'), { code: 'EADDRINUSE' })
++ assert.equal(fixed.bound.length, 1, 'an explicit port gets one attempt')
++ assert.ok(fixed.listeners.every(({ tcp }) => !tcp.listening))
+
- const exhaustedUdp = collidingUdp(Infinity)
- const exhaustedTcp = net.createServer()
- await assert.rejects(listenDnsPair(exhaustedUdp, exhaustedTcp, 0, '127.0.0.1'), {
++ const exhausted = collidingDnsListeners(Infinity)
++ await assert.rejects(listenDnsPair(exhausted.create, 0, '127.0.0.1'), {
+ code: 'EADDRINUSE',
+ })
- assert.equal(exhaustedUdp.bound.length, 20, 'every ephemeral attempt was made')
- assert.equal(exhaustedTcp.listening, false)
++ assert.equal(exhausted.bound.length, 20, 'every ephemeral attempt was made')
++ assert.ok(exhausted.listeners.every(({ tcp }) => !tcp.listening))
+ })
diff --cc tests/e2e/fixtures/wrc-egress-dns-proxy/server.cjs
index 5f3809ad4,d60a105a7..7c8c3c58b
--- a/tests/e2e/fixtures/wrc-egress-dns-proxy/server.cjs
+++ b/tests/e2e/fixtures/wrc-egress-dns-proxy/server.cjs
@@@ -4,6 -4,6 +4,7 @@@ const dgram = require('node:dgram'
const http = require('node:http')
const net = require('node:net')
++const EPHEMERAL_PAIR_ATTEMPTS = 20
const MAX_HELD = 512
const MAX_OBSERVATIONS = 16
const MAX_QUERY_IDENTITIES = 1024
@@@ -138,44 -119,23 +139,44 @@@ async function closeDnsListeners({ udp
})
}
-/**
- * Binds a UDP socket and a TCP server to the same port. TCP and UDP ephemeral
- * ports are allocated independently, so a port free for one transport can be
- * taken for the other: with port 0 the TCP port is chosen first and UDP joins
- * it, choosing a new pair when UDP finds it taken. An explicit port gets one
- * attempt, and running out of attempts rethrows the last EADDRINUSE.
- */
-async function listenDnsPair(udp, tcp, port, host) {
+async function listenDnsPair(createListeners, port, host) {
+ // UDP and TCP have separate port allocations. Hold the TCP reservation
+ // while binding UDP; only an ephemeral opposite-transport collision may
- // choose another candidate, with at most eight fully cleaned-up attempts.
- const attempts = port === 0 ? 8 : 1
++ // choose another candidate, within the existing cleaned-up attempt budget.
+ const attempts = port === 0 ? EPHEMERAL_PAIR_ATTEMPTS : 1
- for (let attempt = 1; ; attempt++) {
- await listenOnce(tcp, () => tcp.listen(port, host))
+ for (let attempt = 0; attempt < attempts; attempt++) {
+ const listeners = createListeners()
+ const { udp, tcp } = listeners
+ const connections = new Set()
+ const connected = socket => connections.add(socket)
+ tcp.on('connection', connected)
+ let firstError
+ const recordError = error => {
+ firstError ??= error
+ }
+ udp.on('error', recordError)
+ tcp.on('error', recordError)
+ let udpCollision = false
try {
- await listenOnce(udp, () => udp.bind(tcp.address().port, host))
- return tcp.address().port
+ await listen(tcp, () => tcp.listen(port, host))
+ try {
+ await listen(udp, () => udp.bind(tcp.address().port, host))
+ } catch (error) {
+ udpCollision = error.code === 'EADDRINUSE' && firstError === error
+ throw error
+ }
+ if (firstError) throw firstError
+ return listeners
} catch (error) {
- await new Promise(resolve => tcp.close(() => resolve()))
- if (error.code !== 'EADDRINUSE' || attempt >= attempts) throw error
+ // A client may arrive before the second transport binds. Closing the
+ // listener alone would wait indefinitely for that accepted connection.
+ for (const socket of connections) socket.destroy()
+ await closeDnsListeners(listeners)
+ if (!udpCollision || attempt + 1 === attempts) throw firstError ?? error
+ } finally {
+ udp.removeListener('error', recordError)
+ tcp.removeListener('error', recordError)
+ tcp.removeListener('connection', connected)
}
}
}