Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
import { createFixtureTls } from '../e2e/_lib/hcc-watch-tls.mjs'
import assert from 'node:assert/strict'
import { spawnSync } from 'node:child_process'
import { X509Certificate } from 'node:crypto'
import { once } from 'node:events'
import {
existsSync,
mkdtempSync,
readFileSync,
rmSync,
symlinkSync,
watch,
writeFileSync,
} from 'node:fs'
import https from 'node:https'
import { tmpdir } from 'node:os'
import { dirname, join, resolve } from 'node:path'
import { before, test } from 'node:test'
import { fileURLToPath } from 'node:url'
import { runInNewContext } from 'node:vm'
import {
createProxy,
proxyUpstreamErrorRecord,
validateCommand,
} from '../e2e/_lib/hcc-watch-api-proxy.mjs'
import { changedEnvironment, restorePatch, snapshot } from '../e2e/_lib/hcc-watch-config.mjs'
const root = resolve(dirname(fileURLToPath(import.meta.url)), '../..')
const allowedPath = '/api/v1/namespaces/mcp-server/services/runtime-witness'
const mcpPath = '/apis/clerum.io/v1alpha1/namespaces/mcp-server/mcpservers?watch=true'
const contextPath = '/apis/clerum.io/v1alpha1/namespaces/mcp-server/contexts?watch=true'
const otherPath = '/api/v1/namespaces/mcp-server/pods?watch=true'
const channelPath = '/apis/clerum.io/v1alpha1/namespaces/channels/communicationchannels'
let front
let upstreamTls
let publicConfig
test('channel hold accepts only a bounded duration', () => {
assert.throws(() => validateCommand({ id: 'hold', action: 'hold-channel', durationMs: 0 }, []))
assert.throws(() => validateCommand({ id: 'hold', action: 'hold-channel', durationMs: 60001 }, []))
assert.deepEqual(validateCommand({ id: 'hold', action: 'hold-channel', durationMs: 60000 }, []),
{ id: 'hold', action: 'hold-channel', durationMs: 60000 })
})
test('channel hold cuts only its watch and keeps Host API requests available', { timeout: 10000 }, async () => {
const controlDir = mkdtempSync(join(tmpdir(), 'hcc-channel-hold-'))
const seen = []
const upstream = https.createServer(upstreamTls, (request, response) => {
seen.push(request.url)
if (request.url.includes('watch=true')) {
response.writeHead(200)
response.flushHeaders()
} else {
response.writeHead(200, { 'content-type': 'application/json' })
response.end('{}')
}
})
let proxy
let watchRequest
const get = path => new Promise((resolveResponse, rejectResponse) => {
https.get({ hostname: '127.0.0.1', port: proxy.server.address().port, path,
servername: 'proxy-fixture.test-fixture.svc', ca: front.cert }, response => {
response.resume()
response.once('end', () => resolveResponse(response.statusCode))
}).once('error', rejectResponse)
})
const ack = async (id, state) => {
for (let attempt = 0; attempt < 50; attempt++) {
if (existsSync(join(controlDir, 'ack.json'))) {
const value = JSON.parse(readFileSync(join(controlDir, 'ack.json'), 'utf8'))
if (value.id === id && value.state === state) return value
}
await new Promise(resolveWait => setTimeout(resolveWait, 20))
}
throw new Error(`missing ${state} acknowledgement`)
}
try {
await new Promise(resolveListen => upstream.listen(0, '127.0.0.1', resolveListen))
proxy = createProxy({ ...front, upstreamCa: upstreamTls.cert,
upstreamHost: '127.0.0.1', upstreamPort: upstream.address().port,
allowedPaths: [], controlDir, periodMs: 60000, minAgeMs: 60000, channelPath })
await new Promise(resolveListen => proxy.server.listen(0, '127.0.0.1', resolveListen))
await new Promise((resolveHeaders, rejectHeaders) => {
watchRequest = https.get({ hostname: '127.0.0.1', port: proxy.server.address().port,
path: `${channelPath}?watch=true`, servername: 'proxy-fixture.test-fixture.svc',
ca: front.cert }, resolveHeaders)
watchRequest.once('error', rejectHeaders)
})
writeFileSync(join(controlDir, 'command.json'), JSON.stringify({
id: 'hold-1', action: 'hold-channel', durationMs: 5000,
}))
const held = await ack('hold-1', 'held')
assert.equal(held.count, 1, 'live CommunicationChannel watch was cut')
const initialHold = JSON.parse(readFileSync(join(controlDir, 'channel-hold.json'), 'utf8'))
assert.equal(await get(channelPath), 503, 'channel LIST remains unavailable')
assert.equal(await get('/apis/clerum.io/v1alpha1/namespaces/mcp-host/hosts/test'), 200)
assert.equal(await get('/api/v1/namespaces/mcp-host/pods'), 200)
assert.equal(seen.filter(path => path.startsWith(channelPath)).length, 1,
'blocked channel retries never reached upstream')
writeFileSync(join(controlDir, 'command.json'), JSON.stringify({
id: 'hold-2', action: 'hold-channel', durationMs: 5000,
}))
const renewed = await ack('hold-2', 'held')
assert.equal(renewed.count, 1, 'renewal retains the original watch-cut witness')
assert.equal(renewed.renewed, true)
assert.equal(renewed.renewals, 1)
assert.ok(renewed.deadlineAtMs > initialHold.deadlineAtMs,
'renewal extends the live hold deadline without a timing sleep')
const renewedHold = JSON.parse(readFileSync(join(controlDir, 'channel-hold.json'), 'utf8'))
assert.deepEqual(renewedHold, {
id: 'hold-2',
state: 'held',
cut: 1,
rejected: 1,
renewals: 1,
deadlineAtMs: renewed.deadlineAtMs,
})
assert.equal(await get(channelPath), 503, 'channel LIST remains unavailable after renewal')
assert.equal(seen.filter(path => path.startsWith(channelPath)).length, 1,
'renewed channel hold still blocks upstream channel requests')
writeFileSync(join(controlDir, 'command.json'), JSON.stringify({
id: 'release-1', action: 'release-channel',
}))
await ack('release-1', 'released')
assert.equal(await get(channelPath), 200, 'authoritative LIST works after release')
} finally {
watchRequest?.destroy()
proxy?.close()
upstream.closeAllConnections()
await new Promise(resolveClose => upstream.close(resolveClose))
rmSync(controlDir, { recursive: true, force: true })
}
})
test(
'quiet upstream WATCH headers reach the client before its first body event',
{ timeout: 5000 },
async () => {
const controlDir = mkdtempSync(join(tmpdir(), 'hcc-quiet-watch-'))
let upstreamResponse
const upstream = https.createServer(upstreamTls, (_request, response) => {
upstreamResponse = response
response.writeHead(200, { 'content-type': 'application/json' })
response.flushHeaders()
})
let proxy
let client
try {
await new Promise(resolveListen => upstream.listen(0, '127.0.0.1', resolveListen))
proxy = createProxy({
...front,
upstreamCa: upstreamTls.cert,
upstreamHost: '127.0.0.1',
upstreamPort: upstream.address().port,
allowedPaths: [],
controlDir,
periodMs: 60000,
minAgeMs: 60000,
})
await new Promise(resolveListen => proxy.server.listen(0, '127.0.0.1', resolveListen))
const headers = new Promise((resolveHeaders, rejectHeaders) => {
client = https.get(
{
hostname: '127.0.0.1',
port: proxy.server.address().port,
path: mcpPath,
servername: 'proxy-fixture.test-fixture.svc',
ca: front.cert,
rejectUnauthorized: true,
},
resolveHeaders
)
client.once('error', rejectHeaders)
})
// No upstream body is emitted until this assertion succeeds. A buffered
// header block therefore cannot pass by riding along with an initial event.
const response = await bounded(headers, 'quiet WATCH headers were not forwarded', 1000)
assert.equal(response.statusCode, 200)
assert.equal(response.headers['content-type'], 'application/json')
const firstEvent = once(response, 'data')
upstreamResponse.write('first-watch-event\n')
const [bytes] = await bounded(firstEvent, 'first WATCH event was not forwarded')
assert.equal(bytes.toString(), 'first-watch-event\n')
} finally {
client?.destroy()
proxy?.close()
upstream.closeAllConnections()
await new Promise(resolveClose => upstream.close(resolveClose))
rmSync(controlDir, { recursive: true, force: true })
}
}
)
test('upstream error records discard messages, headers, bodies and unrecognised codes', () => {
const marker = 'synthetic-sensitive-material-must-not-appear'
for (const code of ['ECONNRESET', 'ERR_TLS_CERT_ALTNAME_INVALID', marker, undefined]) {
const record = proxyUpstreamErrorRecord(
Object.assign(new Error(marker), {
code,
headers: { authorization: marker },
body: marker,
certificate: marker,
})
)
assert.deepEqual(record, {
event: 'hcc-fixture-upstream-error',
code: code === 'ECONNRESET' || code === 'ERR_TLS_CERT_ALTNAME_INVALID' ? code : 'UNKNOWN',
})
assert.equal(JSON.stringify(record).includes(marker), false)
}
})
test('actual positive probe emits only an allowed error code even with sensitive error fields', () => {
const captured = spawnSync(
'/bin/bash',
[
'-c',
`
source "$HELPER"
E2E_HCC_PR_A=0 PROXY_NAME=fixture-proxy HCC_NS=fixture HCC_DEPLOY=fixture
kctl() { case "$1" in get) printf '10.0.0.1';; exec) shift 9; printf '%s' "$1" >&3; exit 0;; *) exit 1;; esac; }
verify_hcc_proxy_network_policy
`,
],
{
env: { ...process.env, HELPER: join(root, 'scripts/e2e/_lib/hcc-watch-recovery-fixture.sh') },
encoding: 'utf8',
stdio: ['ignore', 'pipe', 'pipe', 'pipe'],
timeout: 5000,
}
)
assert.equal(captured.status, 0)
const code = captured.output[3]
assert.ok(code.includes('hcc-fixture-positive-probe-error'))
const marker = 'synthetic-sensitive-material-must-not-appear'
for (const failureCode of ['CERT_HAS_EXPIRED', marker]) {
const records = []
let exitCode
let errorHandler
const request = {
setTimeout() {
return request
},
on(event, handler) {
if (event === 'error') errorHandler = handler
return request
},
end() {
errorHandler({
code: failureCode,
message: marker,
headers: { authorization: marker },
body: marker,
})
},
}
runInNewContext(code, {
require: name =>
name === 'fs' ? { readFileSync: () => Buffer.from(marker) } : { request: () => request },
process: {
argv: ['node', 'fixture-proxy', 'fixture-proxy', ''],
exit: value => {
exitCode = value
},
},
console: { error: value => records.push(JSON.parse(value)) },
setTimeout,
clearTimeout,
Buffer,
})
assert.equal(exitCode, 3)
assert.deepEqual(records, [
{
event: 'hcc-fixture-positive-probe-error',
code: failureCode === 'CERT_HAS_EXPIRED' ? failureCode : 'UNKNOWN',
},
])
assert.equal(JSON.stringify(records).includes(marker), false)
}
// A peer that never ends its response cannot extend the five-second wall
// deadline through activity; expiration is explicit ETIMEDOUT, never success.
let expire
let expiredExit
const expiredRecords = []
const pending = {
on() {
return pending
},
end() {},
}
runInNewContext(code, {
require: name =>
name === 'fs' ? { readFileSync: () => Buffer.from(marker) } : { request: () => pending },
process: {
argv: ['node', 'fixture', 'fixture', ''],
exit: value => {
expiredExit = value
},
},
console: { error: value => expiredRecords.push(JSON.parse(value)) },
Buffer,
setTimeout(callback, ms) {
assert.equal(ms, 5000)
expire = callback
return 1
},
clearTimeout() {},
})
assert.equal(expiredExit, undefined)
expire()
assert.equal(expiredExit, 3)
assert.deepEqual(expiredRecords, [
{ event: 'hcc-fixture-positive-probe-error', code: 'ETIMEDOUT' },
])
})
test('failed positive probe collects only bounded safe proxy event/code records before exit', () => {
const directory = mkdtempSync(join(tmpdir(), 'hcc-proxy-diagnostics-'))
const marker = 'synthetic-sensitive-material-must-not-appear'
try {
const logs = join(directory, 'synthetic-logs')
writeFileSync(
logs,
[
marker,
JSON.stringify({
event: 'hcc-fixture-upstream-error',
code: 'ECONNRESET',
message: marker,
body: marker,
}),
JSON.stringify({ event: 'hcc-fixture-upstream-error', code: marker }),
JSON.stringify({ event: 'unrelated', code: 'ECONNRESET', message: marker }),
].join('\n')
)
const result = spawnSync(
'/bin/bash',
[
'-c',
`
set -euo pipefail
source "$HELPER"
E2E_HCC_PR_A=0 PROXY_NAME=fixture-proxy HCC_NS=fixture HCC_DEPLOY=fixture
kctl() { case "$1" in
get) printf '10.0.0.1';;
exec) return 3;;
logs) [[ "$*" == *--tail=100* && "$*" == *--limit-bytes=32768* ]] || exit 9; cat "$FAKE_LOGS";;
*) exit 1;; esac; }
die() { exit 1; }
verify_hcc_proxy_network_policy
`,
],
{
env: {
...process.env,
HELPER: join(root, 'scripts/e2e/_lib/hcc-watch-recovery-fixture.sh'),
FAKE_LOGS: logs,
},
encoding: 'utf8',
timeout: 5000,
}
)
assert.equal(result.status, 1)
assert.deepEqual(JSON.parse(result.stdout), {
event: 'hcc-fixture-upstream-error',
code: 'ECONNRESET',
})
assert.equal((result.stdout + result.stderr).includes(marker), false)
} finally {
rmSync(directory, { recursive: true, force: true })
}
})
test('projected-file symlink CLI enters startup and fails loud without fixture mounts', () => {
// ConfigMap projection resolves import.meta.url to a timestamped real path,
// while argv[1] retains the projected symlink. Never create root mounts or
// open the proxy listener in this entry-point regression.
assert.equal(existsSync('/fixture-tls'), false, 'test requires no root fixture mount')
const directory = mkdtempSync(join(tmpdir(), 'hcc-proxy-entry-'))
try {
const entry = join(directory, 'projected-proxy.mjs')
symlinkSync(join(root, 'scripts/e2e/_lib/hcc-watch-api-proxy.mjs'), entry)
const result = spawnSync(process.execPath, [entry], {
encoding: 'utf8',
timeout: 5000,
maxBuffer: 65536,
})
assert.equal(result.error, undefined, 'CLI terminated without a runner error')
assert.notEqual(result.status, 0, 'projected CLI must not silently exit successfully')
assert.equal(result.stdout, '')
assert.equal(result.stderr.includes('ENOENT'), true, 'startup attempted its required mount')
assert.equal(
result.stderr.includes('/fixture-tls/tls.key'),
true,
'failure is the expected missing fixture mount'
)
} finally {
rmSync(directory, { recursive: true, force: true })
}
})
function certificate() {
return createFixtureTls('127.0.0.1')
}
before(() => {
const result = spawnSync(
process.execPath,
[
join(root, 'scripts/e2e/_lib/hcc-watch-tls-manifest.mjs'),
'proxy-fixture',
'test-fixture',
'run-fixture',
join(root, 'scripts/e2e/_lib/hcc-watch-api-proxy.mjs'),
],
{
encoding: 'utf8',
timeout: 20000,
maxBuffer: 1048576,
}
)
// Captured generator output stays in memory and is never printed in diagnostics.
assert.equal(result.status, 0, 'fixture manifest generator succeeds')
let manifest
try {
manifest = JSON.parse(result.stdout)
} catch {
throw new Error('fixture generator did not emit JSON')
}
assert.equal(manifest.kind, 'List')
assert.equal(manifest.apiVersion, 'v1')
assert.equal(manifest.items.length, 2)
const tlsResource = manifest.items.find(item => item.kind === 'Secret')
const configMap = manifest.items.find(item => item.kind === 'ConfigMap')
assert.equal(tlsResource?.type, 'kubernetes.io/tls')
assert.equal(tlsResource?.metadata.namespace, 'test-fixture')
assert.equal(tlsResource?.metadata.labels['e2e.clerum.io/run'], 'run-fixture')
assert.equal(typeof tlsResource?.stringData['tls.key'], 'string')
front = { key: tlsResource.stringData['tls.key'], cert: tlsResource.stringData['tls.crt'] }
const cert = new X509Certificate(front.cert)
assert.equal(cert.checkHost('proxy-fixture.test-fixture.svc'), 'proxy-fixture.test-fixture.svc')
assert.equal(cert.ca, true)
publicConfig = JSON.parse(configMap.data['config.json'])
assert.equal(
configMap.data['proxy.mjs'] ===
readFileSync(join(root, 'scripts/e2e/_lib/hcc-watch-api-proxy.mjs'), 'utf8'),
true
)
assert.equal(
JSON.stringify(configMap).includes(JSON.stringify(front.key).slice(1, -1)),
false,
'public config excludes signing material'
)
upstreamTls = certificate()
})
function bounded(promise, message, timeoutMs = 4000) {
let timer
return Promise.race([
promise,
new Promise((_, reject) => {
timer = setTimeout(() => reject(new Error(message)), timeoutMs)
}),
]).finally(() => clearTimeout(timer))
}
function ack(controlDir, id, state, filename = 'ack.json') {
return new Promise((resolveAck, reject) => {
let observer
const done = (error, value) => {
clearTimeout(timer)
observer?.close()
if (error) reject(error)
else resolveAck(value)
}
const read = () => {
let value
try {
value = JSON.parse(readFileSync(join(controlDir, filename), 'utf8'))
} catch {
return
}
if (value.id === id && value.state === state) done(null, value)
}
const timer = setTimeout(
() => done(new Error(`control acknowledgement missing: ${state}`)),
4000
)
observer = watch(controlDir, read)
read()
})
}
async function fixture(run, { wrongCa = false, wrongHostname = false } = {}) {
const upstreamCredentials = wrongHostname ? createFixtureTls('other.invalid') : upstreamTls
const controlDir = mkdtempSync(join(tmpdir(), 'hcc-proxy-test-'))
const requests = []
const responses = new Map()
const upstream = https.createServer(upstreamCredentials, (request, response) => {
requests.push({
path: request.url,
method: request.method,
authorization: request.headers.authorization,
})
if (request.url === '/opaque') {
response.destroy()
return
}
if (request.url.includes('watch=true')) {
responses.set(request.url, response)
response.writeHead(200, { 'content-type': 'application/json' })
response.write('watch-start\n')
} else {
response.writeHead(request.url === '/status' ? 418 : 200, { 'content-type': 'text/plain' })
response.end('upstream-response')
}
})
upstream.on('tlsClientError', () => {})
await new Promise(resolveListen => upstream.listen(0, '127.0.0.1', resolveListen))
const proxy = createProxy({
...front,
upstreamCa: wrongCa ? front.cert : upstreamCredentials.cert,
upstreamHost: '127.0.0.1',
upstreamPort: upstream.address().port,
allowedPaths: [allowedPath],
controlDir,
periodMs: 60000,
minAgeMs: 60000,
})
await new Promise(resolveListen => proxy.server.listen(0, '127.0.0.1', resolveListen))
const clients = new Set()
const request = (path, method = 'GET') => {
let responseResolve
let responseReject
const opened = new Promise((resolveResponse, reject) => {
responseResolve = resolveResponse
responseReject = reject
})
const req = https.request({
hostname: '127.0.0.1',
port: proxy.server.address().port,
servername: 'proxy-fixture.test-fixture.svc',
ca: front.cert,
rejectUnauthorized: true,
method,
path,
headers: { authorization: 'Bearer test' },
agent: false,
})
clients.add(req)
let body = ''
const completed = new Promise((resolveResponse, reject) => {
req.on('response', response => {
responseResolve(response)
response.on('data', chunk => {
body += chunk
})
response.on('end', () => resolveResponse({ status: response.statusCode, body }))
response.on('error', reject)
})
req.on('error', error => {
responseReject(error)
reject(error)
})
})
opened.catch(() => {})
completed.catch(() => {})
req.end()
return {
req,
opened,
completed,
get body() {
return body
},
}
}
const command = async (value, state) => {
const waiting = ack(controlDir, value.id, state)
writeFileSync(join(controlDir, 'command.json'), JSON.stringify(value), { mode: 0o600 })
return waiting
}
let closed = false
const close = async () => {
if (closed) return
closed = true
const stopped = once(proxy.server, 'close')
proxy.close()
await bounded(stopped, 'proxy server did not close')
}
try {
await run({ request, command, controlDir, requests, responses, close })
} finally {
try {
await close()
} finally {
for (const client of clients) client.destroy()
upstream.closeAllConnections()
await bounded(
new Promise(resolveClose => upstream.close(resolveClose)),
'upstream did not close'
)
rmSync(controlDir, { recursive: true, force: true })
}
}
}
async function watchStarted(stream) {
const response = await bounded(stream.opened, 'watch response not opened')
if (!stream.body.includes('watch-start'))
await bounded(once(response, 'data'), 'watch did not flow')
assert.equal(stream.body.includes('watch-start'), true)
return response
}
test('generator emits public CA trust and service-account tokenFile provider', () => {
const cluster = publicConfig.clusters[0].cluster
assert.equal(cluster.server, 'https://proxy-fixture.test-fixture.svc:443')
assert.equal(
Buffer.from(cluster['certificate-authority-data'], 'base64').equals(Buffer.from(front.cert)),
true
)
assert.equal('insecure-skip-tls-verify' in cluster, false)
const user = publicConfig.users[0].user
assert.equal(user['auth-provider'].name, 'tokenFile')
assert.equal(
user['auth-provider'].config.tokenFile,
'/var/run/secrets/kubernetes.io/serviceaccount/token'
)
assert.deepEqual(Object.keys(user), ['auth-provider'])
assert.equal(publicConfig['current-context'], 'fixture')
})
test('commands reject invalid id, method, path, duration and cut scope', () => {
const valid = { action: 'arm', id: 'valid-1', method: 'GET', path: allowedPath, durationMs: 1000 }
assert.equal(validateCommand(valid, [allowedPath]), valid)
for (const change of [
{ id: '../escape' },
{ id: true },
{ id: 123 },
{ method: 'POST' },
{ path: '/api/v1/pods' },
{ durationMs: 999 },
{ durationMs: 25001 },
{ durationMs: 1000.5 },
{ action: 'unknown' },
]) {
assert.throws(() => validateCommand({ ...valid, ...change }, [allowedPath]))
}
assert.throws(() => validateCommand({ action: 'cut', id: 'cut-1', kind: 'Pod' }, [allowedPath]))
})
test(
'verified HTTPS forwards status and synthetic authorization; upstream errors are opaque',
{ timeout: 10000 },
async () => {
await fixture(async ({ request, requests }) => {
const result = await bounded(request('/status').completed, 'short response absent')
assert.equal(result.status, 418)
assert.equal(result.body, 'upstream-response')
assert.equal(requests[0].authorization === 'Bearer test', true)
await assert.rejects(bounded(request('/opaque').completed, 'error response absent'), {
code: 'ECONNRESET',
})
})
}
)
test('wrong upstream CA cannot produce a successful response', { timeout: 10000 }, async () => {
await fixture(
async ({ request, requests }) => {
await assert.rejects(
bounded(request('/status').completed, 'untrusted response did not settle'),
{ code: 'ECONNRESET' }
)
assert.equal(requests.length, 0)
},
{ wrongCa: true }
)
})
test('trusted CA with wrong upstream hostname cannot forward a request', { timeout: 10000 }, async () => {
await fixture(
async ({ request, requests }) => {
await assert.rejects(
bounded(request('/status').completed, 'untrusted response did not settle'),
{ code: 'ECONNRESET' }
)
assert.equal(requests.length, 0)
},
{ wrongHostname: true }
)
})
test(
'exact GET pause acknowledges interception and release while watches keep flowing',
{ timeout: 10000 },
async () => {
await fixture(async ({ request, command, controlDir, requests, responses }) => {
const stream = request(mcpPath)
const response = await watchStarted(stream)
await command(
{ action: 'arm', id: 'arm-release', path: allowedPath, method: 'GET', durationMs: 5000 },
'armed'
)
assert.equal(
(await bounded(request('/status').completed, 'unrelated short request paused')).status,
418
)
assert.equal(
(await bounded(request(allowedPath, 'POST').completed, 'POST was incorrectly paused'))
.status,
200
)
const intercepted = ack(controlDir, 'arm-release', 'intercepted')
const held = request(allowedPath)
await intercepted
assert.equal(
requests.some(value => value.path === allowedPath && value.method === 'GET'),
false
)
const flowing = once(response, 'data')
responses.get(mcpPath).write('watch-during-pause\n')
await bounded(flowing, 'watch stalled behind short request')
assert.equal(stream.body.includes('watch-during-pause'), true)
await command({ action: 'release', id: 'release-1', pauseId: 'arm-release' }, 'released')
assert.equal((await bounded(held.completed, 'held GET did not resume')).status, 200)
assert.equal(
requests.filter(value => value.path === allowedPath && value.method === 'GET').length,
1
)
})
}
)
test(
'minimum pause expires and resumes the actual intercepted request',
{ timeout: 10000 },
async () => {
await fixture(async ({ request, command, controlDir, requests }) => {
await command(
{ action: 'arm', id: 'arm-expiry', path: allowedPath, method: 'GET', durationMs: 1000 },
'armed'
)
const intercepted = ack(controlDir, 'arm-expiry', 'intercepted')
const held = request(allowedPath)
await intercepted
assert.equal(
requests.some(value => value.path === allowedPath && value.method === 'GET'),
false
)
await ack(controlDir, 'arm-expiry', 'expired', 'pause.json')
await command({ action: 'release', id: 'release-expired', pauseId: 'arm-expiry' }, 'rejected')
assert.equal(
JSON.parse(readFileSync(join(controlDir, 'pause.json'), 'utf8')).state,
'expired'
)
assert.equal((await bounded(held.completed, 'expired GET did not resume')).status, 200)
})
}
)
test(
'cut targets only the selected watch kind and preserves short requests and other streams',
{ timeout: 10000 },
async () => {
await fixture(async ({ request, command, controlDir, responses }) => {
const mcp = request(mcpPath)
const context = request(contextPath)
const other = request(otherPath)
await watchStarted(mcp)
const contextResponse = await watchStarted(context)
const otherResponse = await watchStarted(other)
await command(
{ action: 'arm', id: 'arm-cut', path: allowedPath, method: 'GET', durationMs: 5000 },
'armed'
)
const intercepted = ack(controlDir, 'arm-cut', 'intercepted')
const held = request(allowedPath)
await intercepted
const pauseBeforeCut = await ack(controlDir, 'arm-cut', 'intercepted', 'pause.json')
const cut = await command({ action: 'cut', id: 'cut-mcp', kind: 'McpServer' }, 'cut')
assert.deepEqual(
JSON.parse(readFileSync(join(controlDir, 'pause.json'), 'utf8')),
pauseBeforeCut
)
assert.equal(cut.count, 1)
await assert.rejects(bounded(mcp.completed, 'MCP watch did not close'))
for (const [path, response, stream] of [
[contextPath, contextResponse, context],
[otherPath, otherResponse, other],
]) {
const flowing = once(response, 'data')
responses.get(path).write('survived-cut\n')
await bounded(flowing, 'unselected stream was cut')
assert.equal(stream.body.includes('survived-cut'), true)
}
await command({ action: 'release', id: 'release-cut', pauseId: 'arm-cut' }, 'released')
assert.equal((await bounded(held.completed, 'short request was cut')).status, 200)
const both = await command({ action: 'cut', id: 'cut-both', kind: 'both' }, 'cut')
assert.equal(both.count, 1)
await assert.rejects(bounded(context.completed, 'Context watch survived both cut'))
const flowing = once(otherResponse, 'data')
responses.get(otherPath).write('survived-both\n')
await bounded(flowing, 'both cut unrelated watch')
assert.equal(other.body.includes('survived-both'), true)
})
}
)
test(
'close tears down active watches and an armed intercepted request',
{ timeout: 10000 },
async () => {
await fixture(async ({ request, command, controlDir, close }) => {
const stream = request(mcpPath)
await watchStarted(stream)
await command(
{ action: 'arm', id: 'arm-close', path: allowedPath, method: 'GET', durationMs: 25000 },
'armed'
)
const intercepted = ack(controlDir, 'arm-close', 'intercepted')
const held = request(allowedPath)
await intercepted
await close()
await assert.rejects(bounded(stream.completed, 'watch survived shutdown'))
await assert.rejects(bounded(held.completed, 'held request survived shutdown'))
})
}
)
// Apply the emitted JSON Patch to a cloned in-memory Deployment.
function applyPatch(deployment, patches) {
const result = structuredClone(deployment)
for (const patch of patches) {
const keys = patch.path.slice(1).split('/')
const parent = keys.slice(0, -1).reduce((value, key) => value[key], result)
const key = keys.at(-1)
if (patch.op === 'test') assert.deepEqual(parent[key], patch.value)
else if (patch.op === 'add') parent[key] = structuredClone(patch.value)
else if (patch.op === 'remove') delete parent[key]
else throw new Error('unexpected restore operation')
}
return result
}
function deploymentFixture() {
return {
metadata: { uid: 'deployment-fixture-uid', resourceVersion: '1' },
spec: {
template: {
spec: {
containers: [
{ name: 'sidecar', env: [{ name: 'SIDECAR', value: 'untouched' }] },
{ name: 'host-context-controller' },
],
},
},
},
}
}
function injectFixture(deployment) {
const result = structuredClone(deployment)
const pod = result.spec.template.spec
const container = pod.containers.find(value => value.name === 'host-context-controller')
container.env = [
...(container.env ?? []).filter(value => !changedEnvironment.includes(value.name)),
...changedEnvironment.map(name => ({ name, value: 'fixture-value' })),
]
pod.volumes = [
...(pod.volumes ?? []),
{ name: 'hcc-pr-a-config', configMap: { name: 'fixture-config' } },
]
container.volumeMounts = [
...(container.volumeMounts ?? []),
{ name: 'hcc-pr-a-config', mountPath: '/fixture-config' },
]
return result
}
test('config restore removes fixture-added fields that were originally absent', () => {
const original = deploymentFixture()
const saved = snapshot(original)
assert.equal(saved.envPresent, false)
assert.equal(saved.volumesPresent, false)
assert.equal(saved.mountsPresent, false)
assert.equal(changedEnvironment.includes('KUBERNETES_SERVICE_HOST'), false)
assert.equal(changedEnvironment.includes('KUBERNETES_SERVICE_PORT'), false)
assert.equal('aliases' in saved, false)
assert.equal('aliasesPresent' in saved, false)
const restored = applyPatch(injectFixture(original), restorePatch(saved, injectFixture(original)))
assert.deepEqual(restored, original)
})
test('config snapshot is public-only and restore preserves unrelated environment edits', () => {
const original = deploymentFixture()
const pod = original.spec.template.spec
const container = pod.containers[1]
container.env = [
{ name: 'KUBECONFIG', value: '/previous/config' },
{ name: 'UNRELATED', value: 'not-in-public-snapshot' },
]
container.volumeMounts = [{ name: 'existing', mountPath: '/existing' }]
pod.volumes = [{ name: 'existing', emptyDir: {} }]
pod.hostAliases = [{ ip: '127.0.0.3', hostnames: ['original.test'] }]
const saved = snapshot(original)
assert.deepEqual(saved.env, [{ name: 'KUBECONFIG', value: '/previous/config' }])
assert.equal(JSON.stringify(saved).includes('not-in-public-snapshot'), false)
const changed = injectFixture(original)
changed.spec.template.spec.containers[1].env.find(value => value.name === 'UNRELATED').value =
'concurrent-public-edit'
const restored = applyPatch(changed, restorePatch(saved, changed))
const restoredContainer = restored.spec.template.spec.containers[1]
assert.equal(
restoredContainer.env.find(value => value.name === 'UNRELATED').value,
'concurrent-public-edit'
)
assert.equal(
restoredContainer.env.find(value => value.name === 'KUBECONFIG').value,
'/previous/config'
)
assert.deepEqual(snapshot(restored), saved)
assert.deepEqual(
restored.spec.template.spec.containers[0],
original.spec.template.spec.containers[0]
)
})
test('config snapshot and restore reject collisions, identity replacement and unrelated mount changes', () => {
const original = deploymentFixture()
const saved = snapshot(original)
assert.throws(() => snapshot(injectFixture(original)), /fixture_mount_collision/)
const replaced = injectFixture(original)
replaced.metadata.uid = 'different-deployment'
assert.throws(() => restorePatch(saved, replaced), /deployment_uid_changed/)
const removed = injectFixture(original)
removed.spec.template.spec.containers = []
assert.throws(() => restorePatch(saved, removed), /deployment_container_missing/)
for (const field of ['volumes', 'volumeMounts']) {
const changed = injectFixture(original)
const target =
field === 'volumes' ? changed.spec.template.spec : changed.spec.template.spec.containers[1]
target[field].push({ name: 'unrelated-new-mount' })
assert.throws(() => restorePatch(saved, changed), /unrelated_configuration_changed/)
}
})
test('release requires a valid original pause identity', () => {
for (const pauseId of [undefined, '', true, 123, '../wrong']) {
assert.throws(() =>
validateCommand({ action: 'release', id: 'release-invalid', pauseId }, [allowedPath])
)
}
assert.equal(
validateCommand({ action: 'release', id: 'release-valid', pauseId: 'arm-valid' }, [allowedPath])
.pauseId,
'arm-valid'
)
})
test(
'absent or mismatched pause cannot be released or resume an intercepted GET',
{ timeout: 10000 },
async () => {
await fixture(async ({ request, command, controlDir, requests }) => {
await command({ action: 'release', id: 'release-absent', pauseId: 'never-armed' }, 'rejected')
await command(
{ action: 'arm', id: 'arm-identity', path: allowedPath, method: 'GET', durationMs: 5000 },
'armed'
)
await command(
{ action: 'release', id: 'release-not-intercepted', pauseId: 'arm-identity' },
'rejected'
)
assert.equal(JSON.parse(readFileSync(join(controlDir, 'pause.json'), 'utf8')).state, 'armed')
const intercepted = ack(controlDir, 'arm-identity', 'intercepted', 'pause.json')
const held = request(allowedPath)
const ledger = await intercepted
await command(
{ action: 'release', id: 'release-wrong', pauseId: 'different-arm' },
'rejected'
)
assert.deepEqual(JSON.parse(readFileSync(join(controlDir, 'pause.json'), 'utf8')), ledger)
assert.equal(
requests.some(value => value.path === allowedPath),
false
)
await command(
{ action: 'release', id: 'release-correct', pauseId: 'arm-identity' },
'released'
)
const released = await ack(controlDir, 'arm-identity', 'released', 'pause.json')
assert.equal(Number.isFinite(released.at), true)
assert.equal(
(await bounded(held.completed, 'correctly released GET did not resume')).status,
200
)
})
}
)
test('config restore compare-and-swap rejects a newer Deployment revision', () => {
const original = deploymentFixture()
const changed = injectFixture(original)
changed.metadata.resourceVersion = '2'
const missingRevision = structuredClone(changed)
delete missingRevision.metadata.resourceVersion
assert.throws(
() => restorePatch(snapshot(original), missingRevision),
/deployment_resource_version_missing/
)
const patches = restorePatch(snapshot(original), changed)
assert.equal(
patches.some(
patch =>
patch.op === 'test' && patch.path === '/metadata/resourceVersion' && patch.value === '2'
),
true
)
const concurrent = structuredClone(changed)
concurrent.metadata.resourceVersion = '3'
assert.throws(() => applyPatch(concurrent, patches))
assert.equal(applyPatch(changed, patches).metadata.resourceVersion, '2')
})
test('config restoration preserves new host aliases outside fixture ownership', () => {
const original = deploymentFixture()
original.spec.template.spec.hostAliases = [{ ip: '127.0.0.3', hostnames: ['original.test'] }]
const changed = injectFixture(original)
const aliases = [{ ip: '127.0.0.4', hostnames: ['concurrent.test'] }]
changed.spec.template.spec.hostAliases = aliases
const patches = restorePatch(snapshot(original), changed)
assert.equal(
patches.some(patch => patch.path.includes('hostAliases')),
false
)
assert.deepEqual(applyPatch(changed, patches).spec.template.spec.hostAliases, aliases)
})
test('TLS fixture timeout terminates its signer', { timeout: 25000 }, () => {
const directory = mkdtempSync(join(tmpdir(), 'hcc-signer-timeout-'))
const pidPath = join(directory, 'signer.pid')
const signerPath = join(directory, 'signer.cjs')
const quote = value => "'" + value.replaceAll("'", "'\\''") + "'"
let signerPid
try {
writeFileSync(signerPath,
`require('node:fs').writeFileSync(${JSON.stringify(pidPath)}, String(process.pid)); process.stdin.resume(); setTimeout(() => {}, 30000)`)
writeFileSync(join(directory, 'openssl'),
`#!/bin/sh\nif [ "$1" = version ]; then printf "OpenSSL 3.0.0\\n"; exit 0; fi\nexec ${quote(process.execPath)} ${quote(signerPath)}\n`, { mode: 0o700 })
const result = spawnSync(process.execPath, [
join(root, 'scripts/e2e/_lib/hcc-watch-tls-manifest.mjs'),
'timeout-fixture', 'test-fixture', 'run-fixture',
join(root, 'scripts/e2e/_lib/hcc-watch-api-proxy.mjs'),
], { env: { PATH: `${directory}:/usr/bin:/bin` },
encoding: 'utf8', timeout: 22000, maxBuffer: 1048576 })
signerPid = Number(readFileSync(pidPath, 'utf8'))
assert.equal(result.error, undefined, 'the generator enforces its own deadline')
assert.equal(result.status, 1, 'signing timeout fails closed')
assert.equal(result.stdout, '', 'no partial manifest is emitted')
assert.match(result.stderr, /fixture_certificate_generation_failed/)
assert.throws(() => process.kill(signerPid, 0), { code: 'ESRCH' }, 'signer was reaped')
} finally {
if (signerPid) {
try { process.kill(signerPid, 'SIGKILL') } catch (error) {
if (error.code !== 'ESRCH') throw error
}
}
rmSync(directory, { recursive: true, force: true })
}
})