Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
test(hcc): report bounded proxy connection failure codes
Alfredo Lopez4d5aed43feef
diff --git a/scripts/e2e/_lib/hcc-watch-api-proxy.mjs b/scripts/e2e/_lib/hcc-watch-api-proxy.mjsindex 07a1526ec..398b47b47 100644--- a/scripts/e2e/_lib/hcc-watch-api-proxy.mjs+++ b/scripts/e2e/_lib/hcc-watch-api-proxy.mjs@@ -3,6 +3,29 @@ import fs from 'node:fs'import https from 'node:https'import { createBookmarkObservation } from './hcc-watch-bookmarks.mjs'+export function proxyUpstreamErrorRecord(error) {+ const codes = [+ 'ECONNREFUSED',+ 'ECONNRESET',+ 'ETIMEDOUT',+ 'ENOTFOUND',+ 'EAI_AGAIN',+ 'EHOSTUNREACH',+ 'ENETUNREACH',+ 'EPROTO',+ 'CERT_HAS_EXPIRED',+ 'CERT_NOT_YET_VALID',+ 'DEPTH_ZERO_SELF_SIGNED_CERT',+ 'SELF_SIGNED_CERT_IN_CHAIN',+ 'UNABLE_TO_VERIFY_LEAF_SIGNATURE',+ 'UNABLE_TO_GET_ISSUER_CERT_LOCALLY',+ 'ERR_TLS_CERT_ALTNAME_INVALID',+ 'ERR_TLS_HANDSHAKE_TIMEOUT',+ ]+ const code = error?.code+ return { event: 'hcc-fixture-upstream-error', code: codes.includes(code) ? code : 'UNKNOWN' }+}+export function validateCommand(command, allowedPaths) {if (!command || typeof command.id !== 'string' || !/^[a-zA-Z0-9-]{1,80}$/.test(command.id))throw new Error('invalid_control_id')@@ -111,7 +134,10 @@ export function createProxy({incoming.pipe(response)})- upstream.on('error', () => stream.close())+ upstream.on('error', error => {+ console.error(JSON.stringify(proxyUpstreamErrorRecord(error)))+ stream.close()+ })request.pipe(upstream)}if (diff --git a/scripts/e2e/_lib/hcc-watch-recovery-fixture.sh b/scripts/e2e/_lib/hcc-watch-recovery-fixture.shindex 7d813c4f3..744f2d2a9 100755--- a/scripts/e2e/_lib/hcc-watch-recovery-fixture.sh+++ b/scripts/e2e/_lib/hcc-watch-recovery-fixture.sh@@ -164,6 +164,18 @@ EOFdie "Kubernetes API proxy did not become ready"}+hcc_proxy_safe_error_records() {+ # Reconstruct only finite event/code pairs. Raw logs and extra fields never+ # reach the terminal or an artifact, including on a failed initial probe.+ kctl logs "deployment/$PROXY_NAME" -n "$HCC_NS" -c proxy --tail=100 --limit-bytes=32768 2>/dev/null |+ jq -Rrc 'fromjson? | select(.event=="hcc-fixture-upstream-error") | .code as $code |+ select(["ECONNREFUSED","ECONNRESET","ETIMEDOUT","ENOTFOUND","EAI_AGAIN",+ "EHOSTUNREACH","ENETUNREACH","EPROTO","CERT_HAS_EXPIRED","CERT_NOT_YET_VALID",+ "DEPTH_ZERO_SELF_SIGNED_CERT","SELF_SIGNED_CERT_IN_CHAIN","UNABLE_TO_VERIFY_LEAF_SIGNATURE",+ "UNABLE_TO_GET_ISSUER_CERT_LOCALLY","ERR_TLS_CERT_ALTNAME_INVALID","ERR_TLS_HANDSHAKE_TIMEOUT","UNKNOWN"] | index($code)) |+ {event,code}'+}+verify_hcc_proxy_network_policy() {local proxy_dns="${PROXY_NAME}.${HCC_NS}.svc"local proxy_ip positive_probe negative_probe probe_status@@ -176,8 +188,9 @@ verify_hcc_proxy_network_policy() {positive_probe="$(cat <<'NODE'const fs=require('fs'),https=require('https');const root='/var/run/secrets/kubernetes.io/serviceaccount/';+const safeCodes=['ECONNREFUSED','ECONNRESET','ETIMEDOUT','ENOTFOUND','EAI_AGAIN','EHOSTUNREACH','ENETUNREACH','EPROTO','CERT_HAS_EXPIRED','CERT_NOT_YET_VALID','DEPTH_ZERO_SELF_SIGNED_CERT','SELF_SIGNED_CERT_IN_CHAIN','UNABLE_TO_VERIFY_LEAF_SIGNATURE','UNABLE_TO_GET_ISSUER_CERT_LOCALLY','ERR_TLS_CERT_ALTNAME_INVALID','ERR_TLS_HANDSHAKE_TIMEOUT'];const request=https.request({host:process.argv[1],port:443,path:'/version',servername:process.argv[2]||'kubernetes.default.svc',ca:process.argv[3]?Buffer.from(process.argv[3],'base64'):fs.readFileSync(root+'ca.crt'),rejectUnauthorized:true,headers:{authorization:'Bearer '+fs.readFileSync(root+'token','utf8')}},response=>{response.resume();response.on('end',()=>process.exit(response.statusCode===200?0:2))});-request.setTimeout(5000,()=>request.destroy(new Error('timeout')));request.on('error',()=>process.exit(3));request.end();+request.setTimeout(5000,()=>request.destroy(Object.assign(new Error(),{code:'ETIMEDOUT'})));request.on('error',error=>{const code=error?.code;console.error(JSON.stringify({event:'hcc-fixture-positive-probe-error',code:safeCodes.includes(code)?code:'UNKNOWN'}));process.exit(3)});request.end();NODE)"if [ "${E2E_HCC_PR_A:-0}" = 1 ]; then@@ -188,9 +201,11 @@ NODEdie 'PR A proxy public trust configuration missing'positive_servername=$proxy_dnsfi- kctl exec deployment/"$HCC_DEPLOY" -n "$HCC_NS" -c host-context-controller -- \- node -e "$positive_probe" "$proxy_dns" "$positive_servername" "$proxy_public_ca" >/dev/null ||+ if ! kctl exec deployment/"$HCC_DEPLOY" -n "$HCC_NS" -c host-context-controller -- \+ node -e "$positive_probe" "$proxy_dns" "$positive_servername" "$proxy_public_ca" >/dev/null; then+ hcc_proxy_safe_error_records || truedie "HCC cannot reach the Kubernetes API through the isolated proxy"+ fiPROBE_CREATED=1kctl apply -f - >/dev/null <<EOFdiff --git a/scripts/tests/hcc-watch-api-proxy.test.mjs b/scripts/tests/hcc-watch-api-proxy.test.mjsindex 4be604128..2690c99de 100644--- a/scripts/tests/hcc-watch-api-proxy.test.mjs+++ b/scripts/tests/hcc-watch-api-proxy.test.mjs@@ -16,7 +16,12 @@ import { tmpdir } from 'node:os'import { dirname, join, resolve } from 'node:path'import { before, test } from 'node:test'import { fileURLToPath } from 'node:url'-import { createProxy, validateCommand } from '../e2e/_lib/hcc-watch-api-proxy.mjs'+import { runInNewContext } from 'node:vm'+import {+ createProxy,+ proxyUpstreamErrorRecord,+ validateCommand,+} from '../e2e/_lib/hcc-watch-api-proxy.mjs'import { changedEnvironment, restorePatch, snapshot } from '../e2e/_lib/hcc-watch-config.mjs'const root = resolve(dirname(fileURLToPath(import.meta.url)), '../..')@@ -28,6 +33,149 @@ let frontlet upstreamTlslet publicConfig+test('upstream error records discard messages, headers, bodies and unrecognised codes', () => {+ const marker = 'synthetic-sensitive-material-must-not-appear'+ for (const code of ['ECONNRESET', 'ERR_TLS_CERT_ALTNAME_INVALID', marker, undefined]) {+ const record = proxyUpstreamErrorRecord(+ Object.assign(new Error(marker), {+ code,+ headers: { authorization: marker },+ body: marker,+ certificate: marker,+ })+ )+ assert.deepEqual(record, {+ event: 'hcc-fixture-upstream-error',+ code: code === 'ECONNRESET' || code === 'ERR_TLS_CERT_ALTNAME_INVALID' ? code : 'UNKNOWN',+ })+ assert.equal(JSON.stringify(record).includes(marker), false)+ }+})++test('actual positive probe emits only an allowed error code even with sensitive error fields', () => {+ const captured = spawnSync(+ '/bin/bash',+ [+ '-c',+ `+source "$HELPER"+E2E_HCC_PR_A=0 PROXY_NAME=fixture-proxy HCC_NS=fixture HCC_DEPLOY=fixture+kctl() { case "$1" in get) printf '10.0.0.1';; exec) shift 9; printf '%s' "$1" >&3; exit 0;; *) exit 1;; esac; }+verify_hcc_proxy_network_policy+`,+ ],+ {+ env: { ...process.env, HELPER: join(root, 'scripts/e2e/_lib/hcc-watch-recovery-fixture.sh') },+ encoding: 'utf8',+ stdio: ['ignore', 'pipe', 'pipe', 'pipe'],+ timeout: 5000,+ }+ )+ assert.equal(captured.status, 0)+ const code = captured.output[3]+ assert.ok(code.includes('hcc-fixture-positive-probe-error'))+ const marker = 'synthetic-sensitive-material-must-not-appear'+ for (const failureCode of ['CERT_HAS_EXPIRED', marker]) {+ const records = []+ let exitCode+ let errorHandler+ const request = {+ setTimeout() {+ return request+ },+ on(event, handler) {+ if (event === 'error') errorHandler = handler+ return request+ },+ end() {+ errorHandler({+ code: failureCode,+ message: marker,+ headers: { authorization: marker },+ body: marker,+ })+ },+ }+ runInNewContext(code, {+ require: name =>+ name === 'fs' ? { readFileSync: () => Buffer.from(marker) } : { request: () => request },+ process: {+ argv: ['node', 'fixture-proxy', 'fixture-proxy', ''],+ exit: value => {+ exitCode = value+ },+ },+ console: { error: value => records.push(JSON.parse(value)) },+ Buffer,+ })+ assert.equal(exitCode, 3)+ assert.deepEqual(records, [+ {+ event: 'hcc-fixture-positive-probe-error',+ code: failureCode === 'CERT_HAS_EXPIRED' ? failureCode : 'UNKNOWN',+ },+ ])+ assert.equal(JSON.stringify(records).includes(marker), false)+ }+})++test('failed positive probe collects only bounded safe proxy event/code records before exit', () => {+ const directory = mkdtempSync(join(tmpdir(), 'hcc-proxy-diagnostics-'))+ const marker = 'synthetic-sensitive-material-must-not-appear'+ try {+ const logs = join(directory, 'synthetic-logs')+ writeFileSync(+ logs,+ [+ marker,+ JSON.stringify({+ event: 'hcc-fixture-upstream-error',+ code: 'ECONNRESET',+ message: marker,+ body: marker,+ }),+ JSON.stringify({ event: 'hcc-fixture-upstream-error', code: marker }),+ JSON.stringify({ event: 'unrelated', code: 'ECONNRESET', message: marker }),+ ].join('\n')+ )+ const result = spawnSync(+ '/bin/bash',+ [+ '-c',+ `+set -euo pipefail+source "$HELPER"+E2E_HCC_PR_A=0 PROXY_NAME=fixture-proxy HCC_NS=fixture HCC_DEPLOY=fixture+kctl() { case "$1" in+ get) printf '10.0.0.1';;+ exec) return 3;;+ logs) [[ "$*" == *--tail=100* && "$*" == *--limit-bytes=32768* ]] || exit 9; cat "$FAKE_LOGS";;+ *) exit 1;; esac; }+die() { exit 1; }+verify_hcc_proxy_network_policy+`,+ ],+ {+ env: {+ ...process.env,+ HELPER: join(root, 'scripts/e2e/_lib/hcc-watch-recovery-fixture.sh'),+ FAKE_LOGS: logs,+ },+ encoding: 'utf8',+ timeout: 5000,+ }+ )+ assert.equal(result.status, 1)+ assert.deepEqual(JSON.parse(result.stdout), {+ event: 'hcc-fixture-upstream-error',+ code: 'ECONNRESET',+ })+ assert.equal((result.stdout + result.stderr).includes(marker), false)+ } finally {+ rmSync(directory, { recursive: true, force: true })+ }+})+test('projected-file symlink CLI enters startup and fails loud without fixture mounts', () => {// ConfigMap projection resolves import.meta.url to a timestamped real path,// while argv[1] retains the projected symlink. Never create root mounts or