Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
#!/usr/bin/env bash
# Proves that HCC exposes its real readiness and discovery endpoints while the
# initial Host fleet reconciliation is still demonstrably in progress.
#
# The gate temporarily routes HCC's control-api token issuance to an isolated
# in-cluster HTTP blocker, creates a gate-owned fleet while HCC is stopped, and
# restarts HCC. There are more fixture Hosts than two complete bounded worker
# waves, so the initial pass remains deterministically active even though each
# individual held token request has its own client timeout. No real Host or
# runtime-token Secret is modified.
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=scripts/e2e/e2e-lib.sh
source "${SCRIPT_DIR}/e2e-lib.sh"
# shellcheck source=scripts/e2e/_lib/hcc-watch-recovery-lock.sh
# shellcheck disable=SC1091
source "${SCRIPT_DIR}/_lib/hcc-watch-recovery-lock.sh"
# shellcheck source=scripts/e2e/_lib/hcc-watch-recovery-logs.sh
# shellcheck disable=SC1091
source "${SCRIPT_DIR}/_lib/hcc-watch-recovery-logs.sh"
# shellcheck source=scripts/e2e/_lib/hcc-watch-recovery-fixture.sh
# shellcheck disable=SC1091
source "${SCRIPT_DIR}/_lib/hcc-watch-recovery-fixture.sh"
[ -n "$E2E_KUBECONTEXT" ] || {
echo "KUBECONTEXT or E2E_K8S_CONTEXT must select an explicit branch-scoped minikube context." >&2
exit 1
}
is_branch_scoped_e2e_context "$E2E_KUBECONTEXT" || {
echo "Refusing HCC readiness fault injection on non-branch context '${E2E_KUBECONTEXT}'." >&2
exit 1
}
require_safe_kube_context
[ "${E2E_HCC_READINESS_FAULT_INJECTION:-0}" = 1 ] || {
echo "Set E2E_HCC_READINESS_FAULT_INJECTION=1 to acknowledge temporary HCC fault injection." >&2
exit 1
}
command -v jq >/dev/null 2>&1 || {
echo "jq is required" >&2
exit 1
}
HCC_NS="${HCC_NS:-control-plane}"
HCC_DEPLOY="${HCC_DEPLOY:-host-context-controller}"
HCC_GATEWAY_DEPLOY="${HCC_GATEWAY_DEPLOY:-host-context-controller-api-gateway}"
HOST_NS="${MCP_HOST_NS:-mcp-host}"
MCP_NS="${MCP_SERVER_NS:-mcp-server}"
TOKEN_REQUEST_PATH="/api/v1/auth/mcp-host/${HOST_NS}/standalone/tokens"
TOKEN_REQUEST_LOG_PREFIX="holding-token POST ${TOKEN_REQUEST_PATH} host="
RUN_ID="$(date +%s)-$$"
BLOCKER_NAME="$(truncate_rfc1123 "e2e-hcc-readiness-blocker-${RUN_ID}")"
BLOCKER_EGRESS_POLICY_NAME="$(truncate_rfc1123 "${BLOCKER_NAME}-egress")"
SUITE_NAME="hcc-readiness-bootstrap"
FIXTURE_HOST_PREFIX="$(truncate_rfc1123 "e2e-hcc-ready-${RUN_ID}")"
FIXTURE_CONTEXT="$(truncate_rfc1123 "e2e-hcc-ready-context-${RUN_ID}")"
FIXTURE_SECRET="$(truncate_rfc1123 "e2e-hcc-ready-llm-${RUN_ID}")"
START_MARKER='Starting initial Host background convergence'
COMPLETE_MARKER='Completed Host reconciliation after initial Host reconciliation'
FAIL_MARKER='Host reconciliation after initial Host reconciliation failed'
ORIGINAL_CONTROL_API_BASE_URL=""
ORIGINAL_REPLICAS=""
HCC_PORT=""
HOST_RECONCILE_CONCURRENCY=""
FIXTURE_HOST_COUNT=0
FIXTURE_HOST_NAMES=()
HCC_MUTATED=0
BLOCKER_CREATED=0
FIXTURES_CREATED=0
# Read and mutated by the sourced compare-and-swap lock helper.
# shellcheck disable=SC2034
HCC_GATE_LOCK_ACQUIRED=0
# shellcheck disable=SC2034
HCC_GATE_LOCK_NAME=""
# shellcheck disable=SC2034
HCC_GATE_LOCK_UID=""
# shellcheck disable=SC2034
HCC_GATE_FINALIZATION_FAILURE=""
die() {
if [ -n "${new_hcc_pod:-}" ]; then
echo "Recent HCC logs from ${new_hcc_pod}:" >&2
kctl logs "pod/${new_hcc_pod}" -n "$HCC_NS" -c host-context-controller --tail=300 >&2 || true
fi
fail "$*"
exit 1
}
wait_until() {
local timeout=$1 description=$2
shift 2
local deadline now
deadline=$(( $(date +%s) + timeout ))
while :; do
"$@" && return 0
now="$(date +%s)"
[ "$now" -lt "$deadline" ] || break
sleep 1
done
echo "Timed out after ${timeout}s waiting for ${description}" >&2
return 1
}
running_hcc_pod() {
local rows
rows="$(kctl get pods -n "$HCC_NS" -l "app=${HCC_DEPLOY}" \
--field-selector=status.phase=Running \
-o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.metadata.deletionTimestamp}{"\n"}{end}' \
2>/dev/null)" || return 1
awk -F '\t' '$1 != "" && $2 == "" { print $1; exit }' <<<"$rows"
}
hcc_pods_absent() {
local pods
pods="$(kctl get pods -n "$HCC_NS" -l "app=${HCC_DEPLOY}" -o name 2>/dev/null)" ||
return 1
[ -z "$pods" ]
}
hcc_logs() {
local pod=$1
kctl logs "pod/${pod}" -n "$HCC_NS" -c host-context-controller 2>/dev/null
}
hcc_log_contains() {
local pod=$1 marker=$2 logs
logs="$(hcc_logs "$pod")" || return 1
hcc_log_snapshot_contains "$logs" "$marker"
}
blocker_holds_token_request() {
local logs
logs="$(kctl logs "deployment/${BLOCKER_NAME}" -n "$HCC_NS" 2>/dev/null)" || return 1
hcc_log_snapshot_contains "$logs" \
"${TOKEN_REQUEST_LOG_PREFIX}${FIXTURE_HOST_PREFIX}-"
}
blocker_fixture_request_count() {
local logs
logs="$(kctl logs "deployment/${BLOCKER_NAME}" -n "$HCC_NS" 2>/dev/null)" || return 1
awk -v prefix="${TOKEN_REQUEST_LOG_PREFIX}${FIXTURE_HOST_PREFIX}-" '
index($0, prefix) {
host = substr($0, index($0, "host=") + length("host="))
sub(/[[:space:]].*$/, "", host)
if (!(host in seen)) {
seen[host] = 1
count += 1
}
}
END { print count + 0 }
' <<<"$logs"
}
blocker_observed_more_than_one_worker_wave() {
local count
count="$(blocker_fixture_request_count)" || return 1
[ "$count" -gt "$HOST_RECONCILE_CONCURRENCY" ]
}
initial_host_pass_is_active() {
local pod=$1 logs
logs="$(hcc_logs "$pod")" || return 1
hcc_initial_pass_snapshot_is_active \
"$logs" "$START_MARKER" "$COMPLETE_MARKER" "$FAIL_MARKER"
}
control_api_service_excludes_blocker() {
kctl get endpointslice -n "$HCC_NS" -l 'kubernetes.io/service-name=control-api' -o json |
jq -e --arg blocker "$BLOCKER_NAME" \
'all(.items[]?.endpoints[]?; .targetRef.name != $blocker)' >/dev/null
}
readiness_blocker_suite_absent() {
local resources
resources="$(
kctl get deployment,service,networkpolicy -n "$HCC_NS" \
-l 'e2e.clerum.io/hcc-readiness' -o name 2>/dev/null
)" || return 1
[ -z "$resources" ]
}
probe_hcc_ready_pod() {
local pod
pod="$(ready_pod_name "$HCC_NS" "app=${HCC_DEPLOY}")" || return 1
kctl exec "pod/${pod}" -n "$HCC_NS" -c host-context-controller -- \
env "HCC_E2E_PORT=${HCC_PORT}" \
node -e '
const http = require("node:http");
const request = http.get(
{host: "127.0.0.1", port: Number(process.env.HCC_E2E_PORT), path: "/ready"},
response => {
let body = "";
response.setEncoding("utf8");
response.on("data", chunk => { body += chunk; });
response.on("end", () => {
let parsed;
try { parsed = JSON.parse(body); } catch { process.exit(2); }
process.exit(response.statusCode === 200 && parsed?.ready === true ? 0 : 3);
});
}
);
request.setTimeout(5000, () => request.destroy(new Error("timeout")));
request.once("error", () => process.exit(4));
' >/dev/null 2>&1
}
running_hcc_gateway_pod() {
ready_pod_name "$HCC_NS" "app=${HCC_GATEWAY_DEPLOY}"
}
hcc_gateway_deployment_ready() {
kctl get deployment "$HCC_GATEWAY_DEPLOY" -n "$HCC_NS" -o json |
jq -e '
.spec.replicas == 1 and
.status.observedGeneration == .metadata.generation and
(.status.updatedReplicas // 0) == 1 and
(.status.readyReplicas // 0) == 1 and
(.status.availableReplicas // 0) == 1 and
(.status.unavailableReplicas // 0) == 0
' >/dev/null
}
hcc_gateway_local_health_ok() {
local pod
pod="$(running_hcc_gateway_pod)" || return 1
kctl exec "pod/${pod}" -n "$HCC_NS" -c nginx -- \
wget -qO- http://127.0.0.1:8081/health |
jq -e '.status == "ok"' >/dev/null
}
hcc_gateway_ready_proxy_unavailable() {
local pod output status
pod="$(running_hcc_gateway_pod)" || return 1
output="$(
kctl exec "pod/${pod}" -n "$HCC_NS" -c nginx -- \
wget -T 70 -t 1 -O /dev/null -S http://127.0.0.1:8081/ready 2>&1
)" && return 1
status="$(
awk '
/^[[:space:]]*HTTP\/[0-9.]+[[:space:]]+[0-9][0-9][0-9]([[:space:]]|$)/ {
code = $2
}
END {
if (code != "") print code
}
' <<<"$output"
)"
case "$status" in
502|503|504) return 0 ;;
*) return 1 ;;
esac
}
hcc_gateway_remains_ready_without_hcc() {
local duration=$1 deadline
deadline=$(( $(date +%s) + duration ))
while [ "$(date +%s)" -lt "$deadline" ]; do
hcc_gateway_deployment_ready &&
hcc_gateway_local_health_ok &&
hcc_gateway_ready_proxy_unavailable &&
hcc_gateway_deployment_ready &&
hcc_gateway_local_health_ok ||
return 1
sleep 1
done
}
hcc_gateway_ready_proxy_recovers() {
local pod
pod="$(running_hcc_gateway_pod)" || return 1
kctl exec "pod/${pod}" -n "$HCC_NS" -c nginx -- \
wget -qO- http://127.0.0.1:8081/ready |
jq -e '.ready == true and .status == "ready"' >/dev/null
}
fixture_inputs_absent() {
local hosts contexts secrets
hosts="$(kctl get host -n "$HOST_NS" -l "e2e.clerum.io/suite=${SUITE_NAME}" \
-o name 2>/dev/null)" || return 1
contexts="$(kctl get context -n "$MCP_NS" -l "e2e.clerum.io/suite=${SUITE_NAME}" \
-o name 2>/dev/null)" || return 1
secrets="$(kctl get secret -n "$HOST_NS" -l "e2e.clerum.io/suite=${SUITE_NAME}" \
-o name 2>/dev/null)" || return 1
[ -z "${hosts}${contexts}${secrets}" ]
}
fixture_runtime_absent() {
local host resources
[ "${#FIXTURE_HOST_NAMES[@]}" -eq 0 ] && return 0
for host in "${FIXTURE_HOST_NAMES[@]}"; do
resources="$(
kctl get deployment,service,serviceaccount,role,rolebinding,secret,persistentvolumeclaim,networkpolicy \
-A -l "clerum.io/managed-by=host-context-controller,clerum.io/host=${host}" \
-o name 2>/dev/null
)" || return 1
[ -z "$resources" ] || return 1
done
}
fixture_resources_absent() {
fixture_inputs_absent && fixture_runtime_absent
}
hcc_restore_is_verified() {
local deployment
deployment="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o json)" || return 1
jq -e \
--arg controlApiBaseUrl "$ORIGINAL_CONTROL_API_BASE_URL" \
--argjson replicas "${ORIGINAL_REPLICAS:-1}" '
.spec.replicas == $replicas and
.status.observedGeneration == .metadata.generation and
(.status.updatedReplicas // 0) == $replicas and
(.status.readyReplicas // 0) == $replicas and
(.status.availableReplicas // 0) == $replicas and
(.status.unavailableReplicas // 0) == 0 and
any(.spec.template.spec.containers[]?;
.name == "host-context-controller" and
any(.env[]?;
.name == "CONTROL_API_BASE_URL" and .value == $controlApiBaseUrl
)
)
' <<<"$deployment" >/dev/null || return 1
probe_hcc_ready_pod && fixture_resources_absent
}
delete_blocker_fixture() {
local failed=0
kctl delete networkpolicy "$BLOCKER_EGRESS_POLICY_NAME" "$BLOCKER_NAME" -n "$HCC_NS" \
--ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1
kctl delete service "$BLOCKER_NAME" -n "$HCC_NS" \
--ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1
kctl delete deployment "$BLOCKER_NAME" -n "$HCC_NS" \
--ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1
[ "$failed" = 0 ]
}
create_host_fixtures() {
local index host
FIXTURES_CREATED=1
kctl apply -f - >/dev/null <<EOF || return 1
apiVersion: v1
kind: Secret
metadata:
name: ${FIXTURE_SECRET}
namespace: ${HOST_NS}
labels:
e2e.clerum.io/suite: ${SUITE_NAME}
e2e.clerum.io/run: "${RUN_ID}"
type: Opaque
stringData:
# Test-only placeholder: Host secret validation checks existence, while token
# issuance is intentionally held before any mcp-host Deployment is created.
OPENAI_API_KEY: "e2e-readiness-not-a-real-key"
---
apiVersion: clerum.io/v1alpha1
kind: Context
metadata:
name: ${FIXTURE_CONTEXT}
namespace: ${MCP_NS}
labels:
e2e.clerum.io/suite: ${SUITE_NAME}
e2e.clerum.io/run: "${RUN_ID}"
spec:
contextId: ${FIXTURE_CONTEXT}
description: Gate-owned Context for HCC Host bootstrap readiness.
mcpServers: []
EOF
for ((index = 1; index <= FIXTURE_HOST_COUNT; index++)); do
host="$(printf '%s-%02d' "$FIXTURE_HOST_PREFIX" "$index")"
FIXTURE_HOST_NAMES+=("$host")
kctl apply -f - >/dev/null <<EOF || return 1
apiVersion: clerum.io/v1alpha1
kind: Host
metadata:
name: ${host}
namespace: ${HOST_NS}
labels:
e2e.clerum.io/suite: ${SUITE_NAME}
e2e.clerum.io/run: "${RUN_ID}"
spec:
host: ${host}
contextRef: ${FIXTURE_CONTEXT}
secretRef: ${FIXTURE_SECRET}
model:
provider: openai
name: e2e-readiness
EOF
done
}
delete_host_fixtures() {
local failed=0 host
kctl delete host -n "$HOST_NS" -l "e2e.clerum.io/suite=${SUITE_NAME}" \
--ignore-not-found --wait=true --timeout=120s >/dev/null 2>&1 || failed=1
kctl delete context -n "$MCP_NS" -l "e2e.clerum.io/suite=${SUITE_NAME}" \
--ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1
kctl delete secret -n "$HOST_NS" -l "e2e.clerum.io/suite=${SUITE_NAME}" \
--ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1
[ "${#FIXTURE_HOST_NAMES[@]}" -eq 0 ] && return "$failed"
for host in "${FIXTURE_HOST_NAMES[@]}"; do
kctl delete \
deployment,service,serviceaccount,role,rolebinding,secret,persistentvolumeclaim,networkpolicy \
-A -l "clerum.io/managed-by=host-context-controller,clerum.io/host=${host}" \
--ignore-not-found --wait=true --timeout=120s >/dev/null 2>&1 || failed=1
done
[ "$failed" = 0 ] || return 1
wait_until 60 "all gate-owned Host inputs and managed runtime resources to disappear" \
fixture_resources_absent
}
print_repair_instructions() {
cat >&2 <<EOF
HCC readiness bootstrap cleanup could not restore a verified clean state.
Context: ${E2E_KUBECONTEXT}
HCC: ${HCC_NS}/${HCC_DEPLOY}
Fixture Hosts: ${HOST_NS}/${FIXTURE_HOST_PREFIX}-*
Fixture Context: ${MCP_NS}/${FIXTURE_CONTEXT}
Fixture LLM Secret: ${HOST_NS}/${FIXTURE_SECRET}
Retained blocker: ${HCC_NS}/${BLOCKER_NAME}
Inspect before changing anything:
kubectl --context=${E2E_KUBECONTEXT} -n ${HCC_NS} get deployment ${HCC_DEPLOY} -o yaml
kubectl --context=${E2E_KUBECONTEXT} -n ${HOST_NS} get host,secret -l e2e.clerum.io/suite=${SUITE_NAME}
kubectl --context=${E2E_KUBECONTEXT} -n ${MCP_NS} get context -l e2e.clerum.io/suite=${SUITE_NAME}
kubectl --context=${E2E_KUBECONTEXT} -n ${HCC_NS} get deployment,service,networkpolicy -l e2e.clerum.io/hcc-readiness
Do not remove the blocker or HCC gate lock until HCC configuration, rollout, Pod readiness,
/ready, and fixture absence are verified.
Restore the original HCC replica count before verifying rollout and /ready:
kubectl --context=${E2E_KUBECONTEXT} -n ${HCC_NS} scale deployment/${HCC_DEPLOY} --replicas=${ORIGINAL_REPLICAS:-1}
kubectl --context=${E2E_KUBECONTEXT} -n ${HCC_NS} rollout status deployment/${HCC_DEPLOY} --timeout=180s
EOF
}
cleanup() {
local status=$? cleanup_failed=0 restore_ok=1 hcc_stopped=1
trap - EXIT
set +e
# Remove every gate-owned input while HCC is stopped. Restarting HCC with a
# partially deleted fixture fleet would let the real control-api path mutate
# those leftovers and make cleanup non-deterministic.
if [ "$HCC_MUTATED" = 1 ]; then
hcc_stopped=0
kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" --replicas=0 \
>/dev/null 2>&1 || restore_ok=0
if [ "$restore_ok" = 1 ]; then
wait_until 120 "HCC pods to stop before fixture deletion" \
hcc_pods_absent >/dev/null 2>&1 || restore_ok=0
fi
[ "$restore_ok" = 1 ] && hcc_stopped=1
fi
if [ "$FIXTURES_CREATED" = 1 ]; then
if [ "$hcc_stopped" = 1 ]; then
delete_host_fixtures >/dev/null 2>&1 || restore_ok=0
else
restore_ok=0
fi
fi
if [ "$HCC_MUTATED" = 1 ] && [ "$restore_ok" = 1 ] &&
[ -n "$ORIGINAL_CONTROL_API_BASE_URL" ]; then
kctl set env deployment/"$HCC_DEPLOY" -n "$HCC_NS" \
"CONTROL_API_BASE_URL=${ORIGINAL_CONTROL_API_BASE_URL}" >/dev/null 2>&1 ||
restore_ok=0
kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" \
--replicas="${ORIGINAL_REPLICAS:-1}" >/dev/null 2>&1 ||
restore_ok=0
kctl rollout status deployment "$HCC_DEPLOY" -n "$HCC_NS" --timeout=180s \
>/dev/null 2>&1 || restore_ok=0
if [ "$restore_ok" = 1 ]; then
wait_until 240 \
"restored HCC config, rollout, Ready pod, /ready, and fixture absence" \
hcc_restore_is_verified >/dev/null 2>&1 || restore_ok=0
fi
fi
if [ "$restore_ok" = 1 ]; then
if [ "$BLOCKER_CREATED" = 1 ]; then
delete_blocker_fixture || cleanup_failed=1
fi
wait_until 60 "all HCC readiness blocker resources to disappear" \
readiness_blocker_suite_absent >/dev/null 2>&1 || cleanup_failed=1
else
cleanup_failed=1
fi
if ! finalize_hcc_watch_gate_lock "$cleanup_failed" "$restore_ok"; then
cleanup_failed=1
fi
if [ "$cleanup_failed" -ne 0 ]; then
echo "HCC readiness gate cleanup failed on context ${E2E_KUBECONTEXT}." >&2
[ "$restore_ok" = 1 ] || print_repair_instructions
[ "$status" = 0 ] && status=1
fi
if [ "$status" = 0 ] && [ "$cleanup_failed" = 0 ] && [ "$restore_ok" = 1 ]; then
header "HCC readiness bootstrap gate passed"
fi
exit "$status"
}
trap cleanup EXIT
header "HCC readiness during initial Host fleet reconciliation"
require_branch_owned_hcc_gate "$HCC_NS"
ok "branch helper, profile, exact HEAD/fingerprint/gate verified"
kctl get nodes -o json |
jq -e --arg context "$E2E_KUBECONTEXT" \
'any(.items[]; .metadata.labels["minikube.k8s.io/name"] == $context)' >/dev/null ||
die "target context is not a minikube cluster"
readiness_blocker_suite_absent ||
die "stale HCC readiness blocker resources exist; inspect them before fault injection"
fixture_inputs_absent ||
die "stale ${SUITE_NAME} Host, Context, or Secret fixtures exist; inspect them before fault injection"
acquire_hcc_watch_gate_lock ||
die "another disruptive HCC gate owns context ${E2E_KUBECONTEXT}"
ok "exclusive HCC fault-injection lock acquired"
HCC_DEPLOYMENT_JSON="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o json)" ||
die "could not read HCC Deployment"
ORIGINAL_REPLICAS="$(jq -r '.spec.replicas // empty' <<<"$HCC_DEPLOYMENT_JSON")"
[ "$ORIGINAL_REPLICAS" = 1 ] ||
die "expected exactly one HCC replica, found ${ORIGINAL_REPLICAS:-unknown}"
ORIGINAL_CONTROL_API_BASE_URL="$(jq -r '
first(
.spec.template.spec.containers[]? |
select(.name == "host-context-controller") |
.env[]? |
select(.name == "CONTROL_API_BASE_URL") |
.value
) // empty
' <<<"$HCC_DEPLOYMENT_JSON")"
[ -n "$ORIGINAL_CONTROL_API_BASE_URL" ] ||
die "HCC CONTROL_API_BASE_URL is missing"
HCC_PORT="$(jq -r '
.spec.template.spec.containers[]? |
select(.name == "host-context-controller") as $container |
(
[$container.env[]? | select(.name == "CONTEXT_MAPPER_PORT") | .value][0] //
[$container.ports[]? | select(.name == "http") | .containerPort][0] //
empty
)
' <<<"$HCC_DEPLOYMENT_JSON")"
[[ "$HCC_PORT" =~ ^[0-9]+$ ]] && [ "$HCC_PORT" -ge 1 ] && [ "$HCC_PORT" -le 65535 ] ||
die "could not resolve a valid HCC HTTP port from the Deployment"
HOST_RECONCILE_CONCURRENCY="$(jq -r '
first(
.spec.template.spec.containers[]? |
select(.name == "host-context-controller") |
.env[]? |
select(.name == "HCC_HOST_FULL_RECONCILE_CONCURRENCY") |
.value
) // "2"
' <<<"$HCC_DEPLOYMENT_JSON")"
[[ "$HOST_RECONCILE_CONCURRENCY" =~ ^[1-8]$ ]] ||
die "HCC_HOST_FULL_RECONCILE_CONCURRENCY must be 1..8, found ${HOST_RECONCILE_CONCURRENCY}"
FIXTURE_HOST_COUNT=$((HOST_RECONCILE_CONCURRENCY * 2 + 1))
HCC_IMAGE="$(jq -r '
first(
.spec.template.spec.containers[]? |
select(.name == "host-context-controller") |
.image
) // empty
' <<<"$HCC_DEPLOYMENT_JSON")"
[ -n "$HCC_IMAGE" ] || die "could not resolve the HCC image"
ok "planned ${FIXTURE_HOST_COUNT} gate-owned Hosts for concurrency ${HOST_RECONCILE_CONCURRENCY}"
BLOCKER_CREATED=1
kctl apply -f - >/dev/null <<EOF
apiVersion: apps/v1
kind: Deployment
metadata:
name: ${BLOCKER_NAME}
namespace: ${HCC_NS}
labels:
e2e.clerum.io/hcc-readiness: "${RUN_ID}"
spec:
replicas: 1
selector:
matchLabels:
e2e.clerum.io/hcc-readiness: "${RUN_ID}"
template:
metadata:
labels:
e2e.clerum.io/hcc-readiness: "${RUN_ID}"
spec:
automountServiceAccountToken: false
terminationGracePeriodSeconds: 1
containers:
- name: blocker
image: ${HCC_IMAGE}
imagePullPolicy: IfNotPresent
env:
- name: EXPECTED_TOKEN_PATH
value: ${TOKEN_REQUEST_PATH}
command: [node, -e]
args:
- >-
require('http').createServer((request,response) => {
if(request.method!=='POST'||request.url!==process.env.EXPECTED_TOKEN_PATH){
console.log('rejecting '+request.method+' '+request.url);
request.resume();
request.on('end',()=>{response.statusCode=404;response.end()});
return;
}
let body='';
request.setEncoding('utf8');
request.on('data',chunk=>{if(body.length<65536)body+=chunk;});
request.on('end',()=>{
let host='unknown';
try{const parsed=JSON.parse(body);if(typeof parsed.host==='string')
host=parsed.host;}catch{}
console.log('holding-token '+request.method+' '+request.url+' host='+host);
});
}).listen(8090, '0.0.0.0');
readinessProbe:
tcpSocket: {port: 8090}
periodSeconds: 1
resources:
requests: {cpu: 5m, memory: 16Mi}
limits: {cpu: 50m, memory: 48Mi}
securityContext:
allowPrivilegeEscalation: false
capabilities: {drop: [ALL]}
readOnlyRootFilesystem: true
runAsNonRoot: true
runAsUser: 1000
runAsGroup: 1000
seccompProfile: {type: RuntimeDefault}
---
apiVersion: v1
kind: Service
metadata:
name: ${BLOCKER_NAME}
namespace: ${HCC_NS}
labels:
e2e.clerum.io/hcc-readiness: "${RUN_ID}"
spec:
selector:
e2e.clerum.io/hcc-readiness: "${RUN_ID}"
ports:
- name: http
port: 8090
targetPort: 8090
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: ${BLOCKER_EGRESS_POLICY_NAME}
namespace: ${HCC_NS}
labels:
e2e.clerum.io/hcc-readiness: "${RUN_ID}"
spec:
podSelector:
matchLabels:
app: ${HCC_DEPLOY}
policyTypes: [Egress]
egress:
- to:
- podSelector:
matchLabels:
e2e.clerum.io/hcc-readiness: "${RUN_ID}"
ports:
- protocol: TCP
port: 8090
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: ${BLOCKER_NAME}
namespace: ${HCC_NS}
labels:
e2e.clerum.io/hcc-readiness: "${RUN_ID}"
spec:
podSelector:
matchLabels:
e2e.clerum.io/hcc-readiness: "${RUN_ID}"
policyTypes: [Ingress]
ingress:
- from:
- podSelector:
matchLabels:
app: ${HCC_DEPLOY}
ports:
- protocol: TCP
port: 8090
EOF
kctl rollout status deployment "$BLOCKER_NAME" -n "$HCC_NS" --timeout=90s >/dev/null ||
die "isolated token-issuance blocker did not become ready"
control_api_service_excludes_blocker ||
die "isolated blocker was selected by the real control-api Service"
ok "isolated in-cluster blocker is ready"
HCC_MUTATED=1
kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" --replicas=0 >/dev/null
wait_until 120 "existing HCC pods to stop" hcc_pods_absent ||
die "existing HCC pod did not stop"
hcc_gateway_remains_ready_without_hcc 20 ||
die "HCC API gateway did not remain Ready on local /health while proxied /ready was unavailable"
ok "API gateway remained Ready for a complete probe-failure window while /ready reflected HCC unavailability"
create_host_fixtures ||
die "could not create the gate-owned Host fleet while HCC was stopped"
ok "created ${FIXTURE_HOST_COUNT} gate-owned Hosts without modifying any real Host or Secret"
kctl set env deployment/"$HCC_DEPLOY" -n "$HCC_NS" \
"CONTROL_API_BASE_URL=http://${BLOCKER_NAME}.${HCC_NS}.svc.cluster.local:8090" >/dev/null
kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" --replicas=1 >/dev/null
new_hcc_pod=""
capture_running_hcc() {
new_hcc_pod="$(running_hcc_pod)"
[ -n "$new_hcc_pod" ]
}
wait_until 120 "replacement HCC pod to enter Running" capture_running_hcc ||
die "replacement HCC pod did not start"
wait_until 60 "initial Host reconciliation to start" \
hcc_log_contains "$new_hcc_pod" "$START_MARKER" ||
die "HCC never started the initial Host reconciliation"
wait_until 120 "a gate-owned Host token request to reach the blocker" blocker_holds_token_request ||
die "initial Host reconciliation never reached the deterministic blocker"
wait_until 180 "more than one bounded Host worker wave to reach the blocker" \
blocker_observed_more_than_one_worker_wave ||
die "fixture fleet did not prove progress beyond the first bounded Host worker wave"
initial_host_pass_is_active "$new_hcc_pod" ||
die "initial Host reconciliation failed or completed before the readiness assertion"
ok "initial Host reconciliation is active beyond one bounded worker wave"
probe_script="$(cat <<'NODE'
const http = require('http');
function get(path) {
return new Promise((resolve, reject) => {
const request = http.get(
{host: '127.0.0.1', port: Number(process.env.HCC_E2E_PORT), path},
response => {
let body = '';
response.setEncoding('utf8');
response.on('data', chunk => { body += chunk; });
response.on('end', () => resolve({status: response.statusCode, body}));
}
);
request.setTimeout(5000, () => request.destroy(new Error('timeout')));
request.once('error', reject);
});
}
(async () => {
const ready = await get('/ready');
const readyBody = JSON.parse(ready.body);
if (ready.status !== 200 || readyBody.status !== 'ready' || readyBody.ready !== true) {
throw new Error('unexpected readiness response: ' + ready.status + ' ' + ready.body);
}
const discovery = await get('/api/v1/mcpservers');
if (discovery.status !== 200) {
throw new Error('unexpected discovery response: ' + discovery.status);
}
const discoveryBody = JSON.parse(discovery.body);
if (!Array.isArray(discoveryBody.servers) || discoveryBody.contextRef !== '*') {
throw new Error('discovery response does not match the live API contract');
}
console.log(JSON.stringify({readyStatus: ready.status, discoveryStatus: discovery.status,
discoveredServers: discoveryBody.servers.length}));
})().catch(error => { console.error(error.message); process.exit(1); });
NODE
)"
probe_result="$(kctl exec "pod/${new_hcc_pod}" -n "$HCC_NS" -c host-context-controller -- \
env "HCC_E2E_PORT=${HCC_PORT}" node -e "$probe_script")" ||
die "real HCC readiness/discovery HTTP probe failed while the fleet pass was active"
echo "$probe_result" | jq -e \
'.readyStatus == 200 and .discoveryStatus == 200 and (.discoveredServers | type == "number")' \
>/dev/null ||
die "HCC HTTP probe returned an invalid result: ${probe_result}"
hcc_gateway_ready_proxy_recovers ||
die "HCC API gateway /ready did not recover after HCC became ready"
initial_host_pass_is_active "$new_hcc_pod" ||
die "initial Host reconciliation failed or completed before the HTTP assertions finished"
ok "/ready returned 200 while the initial Host fleet pass remained active"
ok "API gateway /ready recovered without changing its local readiness"
ok "/api/v1/mcpservers returned a valid live discovery response during that same pass"
header "HCC readiness assertions passed; restoring branch-owned runtime"
echo "$probe_result"