Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

test(hcc): pin HTTP /metrics wire for unsynced and authority-lost (#447)

Alfredo Lopez43567d9e3ccb
diff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.json
index 665d49863..9f10b8432 100644
--- a/host-context-controller/package-lock.json
+++ b/host-context-controller/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.238",
+ "version": "0.1.239",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-host-context-controller",
- "version": "0.1.238",
+ "version": "0.1.239",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@clerum/image-policy": "file:../packages/image-policy",
diff --git a/host-context-controller/package.json b/host-context-controller/package.json
index 68fdd2e0e..37be548c1 100644
--- a/host-context-controller/package.json
+++ b/host-context-controller/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.238",
+ "version": "0.1.239",
"description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API",
"main": "dist/main.js",
"engines": {
diff --git a/host-context-controller/src/k8sClient.test.ts b/host-context-controller/src/k8sClient.test.ts
index 635323bbe..fd624ce4e 100644
--- a/host-context-controller/src/k8sClient.test.ts
+++ b/host-context-controller/src/k8sClient.test.ts
@@ -67,7 +67,10 @@ async function readLabeledConvergenceMetric(
)
}
-async function requestReadyOverHttp(server: ContextMapperServer): Promise<{
+async function requestHccHttpPath(
+ server: ContextMapperServer,
+ path: '/ready' | '/metrics'
+): Promise<{
statusCode: number | undefined
body: string
}> {
@@ -78,21 +81,50 @@ async function requestReadyOverHttp(server: ContextMapperServer): Promise<{
}
return new Promise((resolve, reject) => {
- const request = http.get(
- { host: '127.0.0.1', port: address.port, path: '/ready' },
- response => {
- let body = ''
- response.setEncoding('utf8')
- response.on('data', chunk => {
- body += chunk
- })
- response.on('end', () => resolve({ statusCode: response.statusCode, body }))
- }
- )
+ const request = http.get({ host: '127.0.0.1', port: address.port, path }, response => {
+ let body = ''
+ response.setEncoding('utf8')
+ response.on('data', chunk => {
+ body += chunk
+ })
+ response.on('end', () => resolve({ statusCode: response.statusCode, body }))
+ })
request.once('error', reject)
})
}
+async function requestReadyOverHttp(server: ContextMapperServer): Promise<{
+ statusCode: number | undefined
+ body: string
+}> {
+ return requestHccHttpPath(server, '/ready')
+}
+
+async function requestMetricsOverHttp(server: ContextMapperServer): Promise<{
+ statusCode: number | undefined
+ body: string
+}> {
+ return requestHccHttpPath(server, '/metrics')
+}
+
+// Label order is not part of the Prometheus contract. Match each label
+// independently so a scrape-path pin cannot pass on registry.get() and fail
+// on the text GKE actually keys.
+function readPromTextMetric(text: string, name: string, labels: Record<string, string>): number {
+ for (const line of text.split('\n')) {
+ if (!line.startsWith(`${name}{`)) continue
+ const close = line.indexOf('}')
+ if (close < 0) continue
+ const labelPart = line.slice(name.length, close + 1)
+ if (!Object.entries(labels).every(([key, value]) => labelPart.includes(`${key}="${value}"`))) {
+ continue
+ }
+ const value = Number(line.slice(close + 1).trim())
+ return Number.isFinite(value) ? value : 0
+ }
+ return 0
+}
+
function stubAuthoritativeInventoryWatch(
watcher: McpServerWatcher,
kind: 'McpServer' | 'Context',
@@ -4010,6 +4042,37 @@ describe('McpServerWatcher startup', () => {
kind: 'context',
})
).toBe(droppedBefore + 1)
+ const authorityServer = new ContextMapperServer(watcher, 0, undefined, undefined, () =>
+ watcher.isReadinessInventoryAuthoritative()
+ )
+ await authorityServer.start()
+ authorityServer.setReady(true)
+ try {
+ const scraped = await requestMetricsOverHttp(authorityServer)
+ expect(scraped.statusCode).toBe(200)
+ expect(
+ readPromTextMetric(scraped.body, 'clerum_hcc_initial_convergence_swallowed_total', {
+ lane: 'NetworkPolicy',
+ sink: 'authority-lost',
+ })
+ ).toBe(swallowedBefore + 1)
+ expect(
+ readPromTextMetric(scraped.body, 'clerum_hcc_initial_convergence_pass_results_total', {
+ lane: 'NetworkPolicy',
+ result: 'aborted-authority',
+ })
+ ).toBe(abortedBefore + 1)
+ expect(
+ readPromTextMetric(
+ scraped.body,
+ 'clerum_hcc_initial_convergence_last_success_timestamp_seconds',
+ { lane: 'NetworkPolicy' }
+ )
+ ).toBe(successTimestampBefore)
+ expect((await requestReadyOverHttp(authorityServer)).statusCode).toBe(503)
+ } finally {
+ await authorityServer.stop()
+ }
;(watcher as any).contexts.clear()
;(watcher as any).servers.clear()
;(watcher as any).contexts.set('current-context', {
@@ -4528,6 +4591,18 @@ describe('McpServerWatcher startup', () => {
'clerum_hcc_initial_convergence_pass_results_total',
{ lane: 'NetworkPolicy', result: 'certified' }
)
+ const deferredBefore = await readLabeledConvergenceMetric(
+ 'clerum_hcc_initial_convergence_pass_results_total',
+ { lane: 'NetworkPolicy', result: 'deferred-unsynced' }
+ )
+ const swallowedBefore = await readLabeledConvergenceMetric(
+ 'clerum_hcc_initial_convergence_swallowed_total',
+ { lane: 'NetworkPolicy', sink: 'unsynced' }
+ )
+ const retriesBefore = await readInitialConvergenceMetric(
+ 'clerum_hcc_initial_convergence_retries_total',
+ 'NetworkPolicy'
+ )
const successTimestampBefore = await readInitialConvergenceMetric(
'clerum_hcc_initial_convergence_last_success_timestamp_seconds',
'NetworkPolicy'
@@ -4546,6 +4621,39 @@ describe('McpServerWatcher startup', () => {
await (watcher as any).runInitialNetworkPolicyConvergence()
expect(mocks.netPolFullReconcile).not.toHaveBeenCalled()
expect((await requestReadyOverHttp(server)).statusCode).toBe(503)
+ const deferredScrape = await requestMetricsOverHttp(server)
+ expect(deferredScrape.statusCode).toBe(200)
+ expect(
+ readPromTextMetric(deferredScrape.body, 'clerum_hcc_initial_convergence_swallowed_total', {
+ lane: 'NetworkPolicy',
+ sink: 'unsynced',
+ })
+ ).toBe(swallowedBefore + 1)
+ expect(
+ readPromTextMetric(
+ deferredScrape.body,
+ 'clerum_hcc_initial_convergence_pass_results_total',
+ {
+ lane: 'NetworkPolicy',
+ result: 'deferred-unsynced',
+ }
+ )
+ ).toBe(deferredBefore + 1)
+ expect(
+ readPromTextMetric(deferredScrape.body, 'clerum_hcc_initial_convergence_retries_total', {
+ lane: 'NetworkPolicy',
+ })
+ ).toBe(retriesBefore + 1)
+ expect(
+ readPromTextMetric(
+ deferredScrape.body,
+ 'clerum_hcc_initial_convergence_pass_results_total',
+ {
+ lane: 'NetworkPolicy',
+ result: 'certified',
+ }
+ )
+ ).toBe(certifiedBefore)
;(watcher as any).mcpServerCacheSynced = true
expect((await requestReadyOverHttp(server)).statusCode).toBe(503)
await vi.advanceTimersByTimeAsync(5000)
@@ -4565,6 +4673,41 @@ describe('McpServerWatcher startup', () => {
)
).toBeGreaterThan(successTimestampBefore)
expect((await requestReadyOverHttp(server)).statusCode).toBe(200)
+ const certifiedScrape = await requestMetricsOverHttp(server)
+ expect(certifiedScrape.statusCode).toBe(200)
+ expect(
+ readPromTextMetric(certifiedScrape.body, 'clerum_hcc_initial_convergence_swallowed_total', {
+ lane: 'NetworkPolicy',
+ sink: 'unsynced',
+ })
+ ).toBe(swallowedBefore + 1)
+ expect(
+ readPromTextMetric(
+ certifiedScrape.body,
+ 'clerum_hcc_initial_convergence_pass_results_total',
+ {
+ lane: 'NetworkPolicy',
+ result: 'deferred-unsynced',
+ }
+ )
+ ).toBe(deferredBefore + 1)
+ expect(
+ readPromTextMetric(
+ certifiedScrape.body,
+ 'clerum_hcc_initial_convergence_pass_results_total',
+ {
+ lane: 'NetworkPolicy',
+ result: 'certified',
+ }
+ )
+ ).toBe(certifiedBefore + 1)
+ expect(
+ readPromTextMetric(
+ certifiedScrape.body,
+ 'clerum_hcc_initial_convergence_last_success_timestamp_seconds',
+ { lane: 'NetworkPolicy' }
+ )
+ ).toBeGreaterThan(successTimestampBefore)
} finally {
mocks.hasCertifiedSafetyInventory.mockReturnValue(true)
mocks.netPolFullReconcile.mockImplementation(async (...args: unknown[]) => {
diff --git a/scripts/e2e/e2e-hcc-np-convergence-ladder.sh b/scripts/e2e/e2e-hcc-np-convergence-ladder.sh
index 159cfe056..ef9aa713a 100755
--- a/scripts/e2e/e2e-hcc-np-convergence-ladder.sh
+++ b/scripts/e2e/e2e-hcc-np-convergence-ladder.sh
@@ -17,6 +17,10 @@
# Anti-vacuity: observe either the unsynced log or the swallowed{unsynced}
# metric increment, AND the retry-schedule log. If the window never appears,
# retry the restart once. Still unseen → exit 3 (HCC_NP_LADDER_VACUOUS).
+# After /ready 200 on that same pod, scrape HTTP /metrics and require
+# deferred-unsynced, swallowed{unsynced}, retries, certified, and last-success
+# to agree on the Prometheus text. That does NOT attribute certification to
+# the 5s timer (watch-recovery may certify first on minikube).
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
CALLER_E2E_KUBECONTEXT="${E2E_KUBECONTEXT:-}"
@@ -32,6 +36,9 @@ fi
source "${SCRIPT_DIR}/_lib/hcc-watch-recovery-lock.sh"
# shellcheck disable=SC1091
source "${SCRIPT_DIR}/_lib/hcc-ready-series.sh"
+# e2e-lib log() writes stdout. This probe captures helper output as pod names,
+# so progress lines must not leak into those substitutions.
+log() { echo -e "${CYAN}[E2E]${NC} $*" >&2; }
OWNED_MINIKUBE_PROFILE="clerum-fix-447-hcc-certification-watchdog-45626152"
VACUOUS_EXIT=3
@@ -90,6 +97,7 @@ VACUOUS=0
ROLLOUT_TRIGGERED=0
OBSERVED_AT=""
OBSERVED_POD=""
+OBSERVED_POD_UID=""
SWALLOW_SOURCE=""
READY_AT=""
ATTEMPT=0
@@ -120,19 +128,79 @@ hcc_pod_metrics() {
wget -T 10 -t 1 -qO- http://127.0.0.1:8081/metrics 2>/dev/null || true
}
-swallowed_unsynced_count() {
- local metrics=$1
- awk '
- $1 ~ /^clerum_hcc_initial_convergence_swallowed_total\{/ &&
- $1 ~ /lane="NetworkPolicy"/ &&
- $1 ~ /sink="unsynced"/ {
- print int($2 + 0)
- found = 1
+# Label order is not part of the Prometheus contract.
+prom_labeled_value() {
+ local metrics=$1 name=$2
+ shift 2
+ local label_spec="" pair
+ for pair in "$@"; do
+ label_spec="${label_spec}${pair}|"
+ done
+ awk -v metric_name="$name" -v label_spec="$label_spec" '
+ BEGIN {
+ n = split(label_spec, required, "|")
+ while (n > 0 && required[n] == "") n--
+ }
+ index($1, metric_name "{") == 1 {
+ ok = 1
+ for (j = 1; j <= n; j++) {
+ if (required[j] == "") continue
+ split(required[j], kv, "=")
+ needle = kv[1] "=\"" kv[2] "\""
+ if (index($1, needle) == 0) ok = 0
+ }
+ if (ok) {
+ print int($NF + 0)
+ found = 1
+ exit
+ }
}
END { if (!found) print 0 }
' <<<"$metrics"
}
+swallowed_unsynced_count() {
+ prom_labeled_value "$1" 'clerum_hcc_initial_convergence_swallowed_total' \
+ 'lane=NetworkPolicy' 'sink=unsynced'
+}
+
+assert_convergence_metrics_closure() {
+ local pod=$1
+ local uid_now metrics deferred swallowed retries certified ts
+ require_hcc_pod_name "$pod"
+ uid_now="$(hcc_pod_uid "$pod")"
+ [ -n "$OBSERVED_POD_UID" ] && [ "$uid_now" = "$OBSERVED_POD_UID" ] ||
+ die "HCC pod uid changed before /metrics closure (${OBSERVED_POD_UID:-none} -> ${uid_now:-none})"
+ metrics="$(hcc_pod_metrics "$pod")"
+ [ -n "$metrics" ] || die "HCC /metrics scrape was empty on ${pod}"
+ deferred="$(prom_labeled_value "$metrics" 'clerum_hcc_initial_convergence_pass_results_total' \
+ 'lane=NetworkPolicy' 'result=deferred-unsynced')"
+ swallowed="$(prom_labeled_value "$metrics" 'clerum_hcc_initial_convergence_swallowed_total' \
+ 'lane=NetworkPolicy' 'sink=unsynced')"
+ retries="$(prom_labeled_value "$metrics" 'clerum_hcc_initial_convergence_retries_total' \
+ 'lane=NetworkPolicy')"
+ certified="$(prom_labeled_value "$metrics" 'clerum_hcc_initial_convergence_pass_results_total' \
+ 'lane=NetworkPolicy' 'result=certified')"
+ ts="$(prom_labeled_value "$metrics" 'clerum_hcc_initial_convergence_last_success_timestamp_seconds' \
+ 'lane=NetworkPolicy')"
+ if [ "${deferred:-0}" -lt 1 ]; then
+ fail "INCONCLUSIVE: no deferred-unsynced pass on ${pod} — /metrics closure is vacuous for #447"
+ return 1
+ fi
+ [ "${swallowed:-0}" = "$deferred" ] &&
+ ok "wire /metrics: swallowed{unsynced}=${swallowed} == deferred-unsynced=${deferred}" ||
+ fail "wire /metrics: swallowed{unsynced}=${swallowed} != deferred-unsynced=${deferred}"
+ [ "${retries:-0}" -ge 1 ] &&
+ ok "wire /metrics: retries_total=${retries} (armed; not attributed to the 5s timer alone)" ||
+ fail "wire /metrics: retries_total=${retries} after a deferred-unsynced pass"
+ [ "${certified:-0}" -ge 1 ] &&
+ ok "wire /metrics: certified=${certified} on the same pod that deferred" ||
+ fail "wire /metrics: certified=${certified} after /ready 200 on a deferred pod"
+ [ "$(printf '%s' "$ts" | awk '{ print ($1 > 0) }')" = 1 ] &&
+ ok "wire /metrics: last_success_timestamp_seconds>0" ||
+ fail "wire /metrics: last_success_timestamp_seconds=${ts} was not set by a certified pass"
+}
+
logs_show_unsynced() {
local logs=$1
grep -Eq 'caches unsynced|deferred: caches unsynced' <<<"$logs"
@@ -147,7 +215,7 @@ write_evidence_artifact() {
{
echo "=== HCC NP convergence ladder ==="
echo "context=${E2E_KUBECONTEXT} attempt=${ATTEMPT} vacuous=${VACUOUS}"
- echo "observed_at=${OBSERVED_AT:-none} swallow_source=${SWALLOW_SOURCE:-none} ready_at=${READY_AT:-none}"
+ echo "observed_at=${OBSERVED_AT:-none} observed_pod=${OBSERVED_POD:-none} observed_uid=${OBSERVED_POD_UID:-none} swallow_source=${SWALLOW_SOURCE:-none} ready_at=${READY_AT:-none}"
echo "=== deployment ==="
kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o wide || true
echo "=== pods ==="
@@ -211,6 +279,7 @@ observe_unsynced_swallow() {
fi
if [ "$seen_swallow" = 1 ] && [ "$seen_retry" = 1 ]; then
OBSERVED_AT="$(date +%s)"
+ OBSERVED_POD_UID="$(hcc_pod_uid "$pod")"
return 0
fi
now="$(date +%s)"
@@ -220,6 +289,12 @@ observe_unsynced_swallow() {
return 1
}
+require_hcc_pod_name() {
+ local pod=$1
+ [[ "$pod" =~ ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ ]] ||
+ die "refusing to use a malformed HCC pod name: ${pod}"
+}
+
restart_hcc_and_wait() {
local old_pod=$1 old_uid=$2
local new_pod new_uid
@@ -229,13 +304,13 @@ restart_hcc_and_wait() {
kctl rollout status "deployment/${HCC_DEPLOY}" -n "$HCC_NS" --timeout="${ROLLOUT_TIMEOUT_SEC}s" >/dev/null ||
die "HCC rollout did not become Ready within ${ROLLOUT_TIMEOUT_SEC}s"
new_pod="$(running_hcc_pod)" || true
- [ -n "$new_pod" ] || die "no Running HCC pod after rollout"
+ require_hcc_pod_name "${new_pod:-}"
new_uid="$(hcc_pod_uid "$new_pod")"
[ -n "$new_uid" ] || die "could not read replacement pod uid"
if [ "$new_pod" = "$old_pod" ] && [ "$new_uid" = "$old_uid" ]; then
die "HCC pod did not change after rollout restart (${old_pod}/${old_uid})"
fi
- printf '%s\n' "$new_pod"
+ NEW_POD="$new_pod"
}
wait_ready_after_observation() {
@@ -278,9 +353,13 @@ old_uid="$(hcc_pod_uid "$old_pod")"
swallow_seen=0
new_pod=""
+NEW_POD=""
+require_hcc_pod_name "$old_pod"
while [ "$ATTEMPT" -lt "$RESTART_ATTEMPTS" ]; do
ATTEMPT=$((ATTEMPT + 1))
- new_pod="$(restart_hcc_and_wait "$old_pod" "$old_uid")"
+ restart_hcc_and_wait "$old_pod" "$old_uid"
+ new_pod="$NEW_POD"
+ require_hcc_pod_name "$new_pod"
if observe_unsynced_swallow "$new_pod"; then
swallow_seen=1
ok "observed unsynced swallow (${SWALLOW_SOURCE}) and retry log on ${new_pod}"
@@ -299,6 +378,7 @@ fi
if wait_ready_after_observation "$new_pod"; then
ok "/ready 200 ${READY_AT}s epoch, $((READY_AT - OBSERVED_AT))s after the observed swallow (budget ${READY_BUDGET_SEC}s)"
+ assert_convergence_metrics_closure "$new_pod"
else
fail "/ready did not become 200 within ${READY_BUDGET_SEC}s after the observed swallow on ${new_pod} (see ${LOG_ARTIFACT})"
fi