Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

test(hcc): report bounded proxy connection failure codes

Alfredo Lopez4d5aed43feef
diff --git a/scripts/e2e/_lib/hcc-watch-api-proxy.mjs b/scripts/e2e/_lib/hcc-watch-api-proxy.mjs
index 07a1526ec..398b47b47 100644
--- a/scripts/e2e/_lib/hcc-watch-api-proxy.mjs
+++ b/scripts/e2e/_lib/hcc-watch-api-proxy.mjs
@@ -3,6 +3,29 @@ import fs from 'node:fs'
import https from 'node:https'
import { createBookmarkObservation } from './hcc-watch-bookmarks.mjs'
+export function proxyUpstreamErrorRecord(error) {
+ const codes = [
+ 'ECONNREFUSED',
+ 'ECONNRESET',
+ 'ETIMEDOUT',
+ 'ENOTFOUND',
+ 'EAI_AGAIN',
+ 'EHOSTUNREACH',
+ 'ENETUNREACH',
+ 'EPROTO',
+ 'CERT_HAS_EXPIRED',
+ 'CERT_NOT_YET_VALID',
+ 'DEPTH_ZERO_SELF_SIGNED_CERT',
+ 'SELF_SIGNED_CERT_IN_CHAIN',
+ 'UNABLE_TO_VERIFY_LEAF_SIGNATURE',
+ 'UNABLE_TO_GET_ISSUER_CERT_LOCALLY',
+ 'ERR_TLS_CERT_ALTNAME_INVALID',
+ 'ERR_TLS_HANDSHAKE_TIMEOUT',
+ ]
+ const code = error?.code
+ return { event: 'hcc-fixture-upstream-error', code: codes.includes(code) ? code : 'UNKNOWN' }
+}
+
export function validateCommand(command, allowedPaths) {
if (!command || typeof command.id !== 'string' || !/^[a-zA-Z0-9-]{1,80}$/.test(command.id))
throw new Error('invalid_control_id')
@@ -111,7 +134,10 @@ export function createProxy({
incoming.pipe(response)
}
)
- upstream.on('error', () => stream.close())
+ upstream.on('error', error => {
+ console.error(JSON.stringify(proxyUpstreamErrorRecord(error)))
+ stream.close()
+ })
request.pipe(upstream)
}
if (
diff --git a/scripts/e2e/_lib/hcc-watch-recovery-fixture.sh b/scripts/e2e/_lib/hcc-watch-recovery-fixture.sh
index 7d813c4f3..744f2d2a9 100755
--- a/scripts/e2e/_lib/hcc-watch-recovery-fixture.sh
+++ b/scripts/e2e/_lib/hcc-watch-recovery-fixture.sh
@@ -164,6 +164,18 @@ EOF
die "Kubernetes API proxy did not become ready"
}
+hcc_proxy_safe_error_records() {
+ # Reconstruct only finite event/code pairs. Raw logs and extra fields never
+ # reach the terminal or an artifact, including on a failed initial probe.
+ kctl logs "deployment/$PROXY_NAME" -n "$HCC_NS" -c proxy --tail=100 --limit-bytes=32768 2>/dev/null |
+ jq -Rrc 'fromjson? | select(.event=="hcc-fixture-upstream-error") | .code as $code |
+ select(["ECONNREFUSED","ECONNRESET","ETIMEDOUT","ENOTFOUND","EAI_AGAIN",
+ "EHOSTUNREACH","ENETUNREACH","EPROTO","CERT_HAS_EXPIRED","CERT_NOT_YET_VALID",
+ "DEPTH_ZERO_SELF_SIGNED_CERT","SELF_SIGNED_CERT_IN_CHAIN","UNABLE_TO_VERIFY_LEAF_SIGNATURE",
+ "UNABLE_TO_GET_ISSUER_CERT_LOCALLY","ERR_TLS_CERT_ALTNAME_INVALID","ERR_TLS_HANDSHAKE_TIMEOUT","UNKNOWN"] | index($code)) |
+ {event,code}'
+}
+
verify_hcc_proxy_network_policy() {
local proxy_dns="${PROXY_NAME}.${HCC_NS}.svc"
local proxy_ip positive_probe negative_probe probe_status
@@ -176,8 +188,9 @@ verify_hcc_proxy_network_policy() {
positive_probe="$(cat <<'NODE'
const fs=require('fs'),https=require('https');
const root='/var/run/secrets/kubernetes.io/serviceaccount/';
+const safeCodes=['ECONNREFUSED','ECONNRESET','ETIMEDOUT','ENOTFOUND','EAI_AGAIN','EHOSTUNREACH','ENETUNREACH','EPROTO','CERT_HAS_EXPIRED','CERT_NOT_YET_VALID','DEPTH_ZERO_SELF_SIGNED_CERT','SELF_SIGNED_CERT_IN_CHAIN','UNABLE_TO_VERIFY_LEAF_SIGNATURE','UNABLE_TO_GET_ISSUER_CERT_LOCALLY','ERR_TLS_CERT_ALTNAME_INVALID','ERR_TLS_HANDSHAKE_TIMEOUT'];
const request=https.request({host:process.argv[1],port:443,path:'/version',servername:process.argv[2]||'kubernetes.default.svc',ca:process.argv[3]?Buffer.from(process.argv[3],'base64'):fs.readFileSync(root+'ca.crt'),rejectUnauthorized:true,headers:{authorization:'Bearer '+fs.readFileSync(root+'token','utf8')}},response=>{response.resume();response.on('end',()=>process.exit(response.statusCode===200?0:2))});
-request.setTimeout(5000,()=>request.destroy(new Error('timeout')));request.on('error',()=>process.exit(3));request.end();
+request.setTimeout(5000,()=>request.destroy(Object.assign(new Error(),{code:'ETIMEDOUT'})));request.on('error',error=>{const code=error?.code;console.error(JSON.stringify({event:'hcc-fixture-positive-probe-error',code:safeCodes.includes(code)?code:'UNKNOWN'}));process.exit(3)});request.end();
NODE
)"
if [ "${E2E_HCC_PR_A:-0}" = 1 ]; then
@@ -188,9 +201,11 @@ NODE
die 'PR A proxy public trust configuration missing'
positive_servername=$proxy_dns
fi
- kctl exec deployment/"$HCC_DEPLOY" -n "$HCC_NS" -c host-context-controller -- \
- node -e "$positive_probe" "$proxy_dns" "$positive_servername" "$proxy_public_ca" >/dev/null ||
+ if ! kctl exec deployment/"$HCC_DEPLOY" -n "$HCC_NS" -c host-context-controller -- \
+ node -e "$positive_probe" "$proxy_dns" "$positive_servername" "$proxy_public_ca" >/dev/null; then
+ hcc_proxy_safe_error_records || true
die "HCC cannot reach the Kubernetes API through the isolated proxy"
+ fi
PROBE_CREATED=1
kctl apply -f - >/dev/null <<EOF
diff --git a/scripts/tests/hcc-watch-api-proxy.test.mjs b/scripts/tests/hcc-watch-api-proxy.test.mjs
index 4be604128..2690c99de 100644
--- a/scripts/tests/hcc-watch-api-proxy.test.mjs
+++ b/scripts/tests/hcc-watch-api-proxy.test.mjs
@@ -16,7 +16,12 @@ import { tmpdir } from 'node:os'
import { dirname, join, resolve } from 'node:path'
import { before, test } from 'node:test'
import { fileURLToPath } from 'node:url'
-import { createProxy, validateCommand } from '../e2e/_lib/hcc-watch-api-proxy.mjs'
+import { runInNewContext } from 'node:vm'
+import {
+ createProxy,
+ proxyUpstreamErrorRecord,
+ validateCommand,
+} from '../e2e/_lib/hcc-watch-api-proxy.mjs'
import { changedEnvironment, restorePatch, snapshot } from '../e2e/_lib/hcc-watch-config.mjs'
const root = resolve(dirname(fileURLToPath(import.meta.url)), '../..')
@@ -28,6 +33,149 @@ let front
let upstreamTls
let publicConfig
+test('upstream error records discard messages, headers, bodies and unrecognised codes', () => {
+ const marker = 'synthetic-sensitive-material-must-not-appear'
+ for (const code of ['ECONNRESET', 'ERR_TLS_CERT_ALTNAME_INVALID', marker, undefined]) {
+ const record = proxyUpstreamErrorRecord(
+ Object.assign(new Error(marker), {
+ code,
+ headers: { authorization: marker },
+ body: marker,
+ certificate: marker,
+ })
+ )
+ assert.deepEqual(record, {
+ event: 'hcc-fixture-upstream-error',
+ code: code === 'ECONNRESET' || code === 'ERR_TLS_CERT_ALTNAME_INVALID' ? code : 'UNKNOWN',
+ })
+ assert.equal(JSON.stringify(record).includes(marker), false)
+ }
+})
+
+test('actual positive probe emits only an allowed error code even with sensitive error fields', () => {
+ const captured = spawnSync(
+ '/bin/bash',
+ [
+ '-c',
+ `
+source "$HELPER"
+E2E_HCC_PR_A=0 PROXY_NAME=fixture-proxy HCC_NS=fixture HCC_DEPLOY=fixture
+kctl() { case "$1" in get) printf '10.0.0.1';; exec) shift 9; printf '%s' "$1" >&3; exit 0;; *) exit 1;; esac; }
+verify_hcc_proxy_network_policy
+`,
+ ],
+ {
+ env: { ...process.env, HELPER: join(root, 'scripts/e2e/_lib/hcc-watch-recovery-fixture.sh') },
+ encoding: 'utf8',
+ stdio: ['ignore', 'pipe', 'pipe', 'pipe'],
+ timeout: 5000,
+ }
+ )
+ assert.equal(captured.status, 0)
+ const code = captured.output[3]
+ assert.ok(code.includes('hcc-fixture-positive-probe-error'))
+ const marker = 'synthetic-sensitive-material-must-not-appear'
+ for (const failureCode of ['CERT_HAS_EXPIRED', marker]) {
+ const records = []
+ let exitCode
+ let errorHandler
+ const request = {
+ setTimeout() {
+ return request
+ },
+ on(event, handler) {
+ if (event === 'error') errorHandler = handler
+ return request
+ },
+ end() {
+ errorHandler({
+ code: failureCode,
+ message: marker,
+ headers: { authorization: marker },
+ body: marker,
+ })
+ },
+ }
+ runInNewContext(code, {
+ require: name =>
+ name === 'fs' ? { readFileSync: () => Buffer.from(marker) } : { request: () => request },
+ process: {
+ argv: ['node', 'fixture-proxy', 'fixture-proxy', ''],
+ exit: value => {
+ exitCode = value
+ },
+ },
+ console: { error: value => records.push(JSON.parse(value)) },
+ Buffer,
+ })
+ assert.equal(exitCode, 3)
+ assert.deepEqual(records, [
+ {
+ event: 'hcc-fixture-positive-probe-error',
+ code: failureCode === 'CERT_HAS_EXPIRED' ? failureCode : 'UNKNOWN',
+ },
+ ])
+ assert.equal(JSON.stringify(records).includes(marker), false)
+ }
+})
+
+test('failed positive probe collects only bounded safe proxy event/code records before exit', () => {
+ const directory = mkdtempSync(join(tmpdir(), 'hcc-proxy-diagnostics-'))
+ const marker = 'synthetic-sensitive-material-must-not-appear'
+ try {
+ const logs = join(directory, 'synthetic-logs')
+ writeFileSync(
+ logs,
+ [
+ marker,
+ JSON.stringify({
+ event: 'hcc-fixture-upstream-error',
+ code: 'ECONNRESET',
+ message: marker,
+ body: marker,
+ }),
+ JSON.stringify({ event: 'hcc-fixture-upstream-error', code: marker }),
+ JSON.stringify({ event: 'unrelated', code: 'ECONNRESET', message: marker }),
+ ].join('\n')
+ )
+ const result = spawnSync(
+ '/bin/bash',
+ [
+ '-c',
+ `
+set -euo pipefail
+source "$HELPER"
+E2E_HCC_PR_A=0 PROXY_NAME=fixture-proxy HCC_NS=fixture HCC_DEPLOY=fixture
+kctl() { case "$1" in
+ get) printf '10.0.0.1';;
+ exec) return 3;;
+ logs) [[ "$*" == *--tail=100* && "$*" == *--limit-bytes=32768* ]] || exit 9; cat "$FAKE_LOGS";;
+ *) exit 1;; esac; }
+die() { exit 1; }
+verify_hcc_proxy_network_policy
+`,
+ ],
+ {
+ env: {
+ ...process.env,
+ HELPER: join(root, 'scripts/e2e/_lib/hcc-watch-recovery-fixture.sh'),
+ FAKE_LOGS: logs,
+ },
+ encoding: 'utf8',
+ timeout: 5000,
+ }
+ )
+ assert.equal(result.status, 1)
+ assert.deepEqual(JSON.parse(result.stdout), {
+ event: 'hcc-fixture-upstream-error',
+ code: 'ECONNRESET',
+ })
+ assert.equal((result.stdout + result.stderr).includes(marker), false)
+ } finally {
+ rmSync(directory, { recursive: true, force: true })
+ }
+})
+
test('projected-file symlink CLI enters startup and fails loud without fixture mounts', () => {
// ConfigMap projection resolves import.meta.url to a timestamped real path,
// while argv[1] retains the projected symlink. Never create root mounts or