Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
test(hcc): pin HTTP /metrics wire for unsynced and authority-lost (#447)
Alfredo Lopez43567d9e3ccb
diff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.jsonindex 665d49863..9f10b8432 100644--- a/host-context-controller/package-lock.json+++ b/host-context-controller/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-host-context-controller",- "version": "0.1.238",+ "version": "0.1.239","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-host-context-controller",- "version": "0.1.238",+ "version": "0.1.239","license": "SEE LICENSE IN LICENSE","dependencies": {"@clerum/image-policy": "file:../packages/image-policy",diff --git a/host-context-controller/package.json b/host-context-controller/package.jsonindex 68fdd2e0e..37be548c1 100644--- a/host-context-controller/package.json+++ b/host-context-controller/package.json@@ -1,6 +1,6 @@{"name": "clerum-host-context-controller",- "version": "0.1.238",+ "version": "0.1.239","description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API","main": "dist/main.js","engines": {diff --git a/host-context-controller/src/k8sClient.test.ts b/host-context-controller/src/k8sClient.test.tsindex 635323bbe..fd624ce4e 100644--- a/host-context-controller/src/k8sClient.test.ts+++ b/host-context-controller/src/k8sClient.test.ts@@ -67,7 +67,10 @@ async function readLabeledConvergenceMetric()}-async function requestReadyOverHttp(server: ContextMapperServer): Promise<{+async function requestHccHttpPath(+ server: ContextMapperServer,+ path: '/ready' | '/metrics'+): Promise<{statusCode: number | undefinedbody: string}> {@@ -78,21 +81,50 @@ async function requestReadyOverHttp(server: ContextMapperServer): Promise<{}return new Promise((resolve, reject) => {- const request = http.get(- { host: '127.0.0.1', port: address.port, path: '/ready' },- response => {- let body = ''- response.setEncoding('utf8')- response.on('data', chunk => {- body += chunk- })- response.on('end', () => resolve({ statusCode: response.statusCode, body }))- }- )+ const request = http.get({ host: '127.0.0.1', port: address.port, path }, response => {+ let body = ''+ response.setEncoding('utf8')+ response.on('data', chunk => {+ body += chunk+ })+ response.on('end', () => resolve({ statusCode: response.statusCode, body }))+ })request.once('error', reject)})}+async function requestReadyOverHttp(server: ContextMapperServer): Promise<{+ statusCode: number | undefined+ body: string+}> {+ return requestHccHttpPath(server, '/ready')+}++async function requestMetricsOverHttp(server: ContextMapperServer): Promise<{+ statusCode: number | undefined+ body: string+}> {+ return requestHccHttpPath(server, '/metrics')+}++// Label order is not part of the Prometheus contract. Match each label+// independently so a scrape-path pin cannot pass on registry.get() and fail+// on the text GKE actually keys.+function readPromTextMetric(text: string, name: string, labels: Record<string, string>): number {+ for (const line of text.split('\n')) {+ if (!line.startsWith(`${name}{`)) continue+ const close = line.indexOf('}')+ if (close < 0) continue+ const labelPart = line.slice(name.length, close + 1)+ if (!Object.entries(labels).every(([key, value]) => labelPart.includes(`${key}="${value}"`))) {+ continue+ }+ const value = Number(line.slice(close + 1).trim())+ return Number.isFinite(value) ? value : 0+ }+ return 0+}+function stubAuthoritativeInventoryWatch(watcher: McpServerWatcher,kind: 'McpServer' | 'Context',@@ -4010,6 +4042,37 @@ describe('McpServerWatcher startup', () => {kind: 'context',})).toBe(droppedBefore + 1)+ const authorityServer = new ContextMapperServer(watcher, 0, undefined, undefined, () =>+ watcher.isReadinessInventoryAuthoritative()+ )+ await authorityServer.start()+ authorityServer.setReady(true)+ try {+ const scraped = await requestMetricsOverHttp(authorityServer)+ expect(scraped.statusCode).toBe(200)+ expect(+ readPromTextMetric(scraped.body, 'clerum_hcc_initial_convergence_swallowed_total', {+ lane: 'NetworkPolicy',+ sink: 'authority-lost',+ })+ ).toBe(swallowedBefore + 1)+ expect(+ readPromTextMetric(scraped.body, 'clerum_hcc_initial_convergence_pass_results_total', {+ lane: 'NetworkPolicy',+ result: 'aborted-authority',+ })+ ).toBe(abortedBefore + 1)+ expect(+ readPromTextMetric(+ scraped.body,+ 'clerum_hcc_initial_convergence_last_success_timestamp_seconds',+ { lane: 'NetworkPolicy' }+ )+ ).toBe(successTimestampBefore)+ expect((await requestReadyOverHttp(authorityServer)).statusCode).toBe(503)+ } finally {+ await authorityServer.stop()+ };(watcher as any).contexts.clear();(watcher as any).servers.clear();(watcher as any).contexts.set('current-context', {@@ -4528,6 +4591,18 @@ describe('McpServerWatcher startup', () => {'clerum_hcc_initial_convergence_pass_results_total',{ lane: 'NetworkPolicy', result: 'certified' })+ const deferredBefore = await readLabeledConvergenceMetric(+ 'clerum_hcc_initial_convergence_pass_results_total',+ { lane: 'NetworkPolicy', result: 'deferred-unsynced' }+ )+ const swallowedBefore = await readLabeledConvergenceMetric(+ 'clerum_hcc_initial_convergence_swallowed_total',+ { lane: 'NetworkPolicy', sink: 'unsynced' }+ )+ const retriesBefore = await readInitialConvergenceMetric(+ 'clerum_hcc_initial_convergence_retries_total',+ 'NetworkPolicy'+ )const successTimestampBefore = await readInitialConvergenceMetric('clerum_hcc_initial_convergence_last_success_timestamp_seconds','NetworkPolicy'@@ -4546,6 +4621,39 @@ describe('McpServerWatcher startup', () => {await (watcher as any).runInitialNetworkPolicyConvergence()expect(mocks.netPolFullReconcile).not.toHaveBeenCalled()expect((await requestReadyOverHttp(server)).statusCode).toBe(503)+ const deferredScrape = await requestMetricsOverHttp(server)+ expect(deferredScrape.statusCode).toBe(200)+ expect(+ readPromTextMetric(deferredScrape.body, 'clerum_hcc_initial_convergence_swallowed_total', {+ lane: 'NetworkPolicy',+ sink: 'unsynced',+ })+ ).toBe(swallowedBefore + 1)+ expect(+ readPromTextMetric(+ deferredScrape.body,+ 'clerum_hcc_initial_convergence_pass_results_total',+ {+ lane: 'NetworkPolicy',+ result: 'deferred-unsynced',+ }+ )+ ).toBe(deferredBefore + 1)+ expect(+ readPromTextMetric(deferredScrape.body, 'clerum_hcc_initial_convergence_retries_total', {+ lane: 'NetworkPolicy',+ })+ ).toBe(retriesBefore + 1)+ expect(+ readPromTextMetric(+ deferredScrape.body,+ 'clerum_hcc_initial_convergence_pass_results_total',+ {+ lane: 'NetworkPolicy',+ result: 'certified',+ }+ )+ ).toBe(certifiedBefore);(watcher as any).mcpServerCacheSynced = trueexpect((await requestReadyOverHttp(server)).statusCode).toBe(503)await vi.advanceTimersByTimeAsync(5000)@@ -4565,6 +4673,41 @@ describe('McpServerWatcher startup', () => {)).toBeGreaterThan(successTimestampBefore)expect((await requestReadyOverHttp(server)).statusCode).toBe(200)+ const certifiedScrape = await requestMetricsOverHttp(server)+ expect(certifiedScrape.statusCode).toBe(200)+ expect(+ readPromTextMetric(certifiedScrape.body, 'clerum_hcc_initial_convergence_swallowed_total', {+ lane: 'NetworkPolicy',+ sink: 'unsynced',+ })+ ).toBe(swallowedBefore + 1)+ expect(+ readPromTextMetric(+ certifiedScrape.body,+ 'clerum_hcc_initial_convergence_pass_results_total',+ {+ lane: 'NetworkPolicy',+ result: 'deferred-unsynced',+ }+ )+ ).toBe(deferredBefore + 1)+ expect(+ readPromTextMetric(+ certifiedScrape.body,+ 'clerum_hcc_initial_convergence_pass_results_total',+ {+ lane: 'NetworkPolicy',+ result: 'certified',+ }+ )+ ).toBe(certifiedBefore + 1)+ expect(+ readPromTextMetric(+ certifiedScrape.body,+ 'clerum_hcc_initial_convergence_last_success_timestamp_seconds',+ { lane: 'NetworkPolicy' }+ )+ ).toBeGreaterThan(successTimestampBefore)} finally {mocks.hasCertifiedSafetyInventory.mockReturnValue(true)mocks.netPolFullReconcile.mockImplementation(async (...args: unknown[]) => {diff --git a/scripts/e2e/e2e-hcc-np-convergence-ladder.sh b/scripts/e2e/e2e-hcc-np-convergence-ladder.shindex 159cfe056..ef9aa713a 100755--- a/scripts/e2e/e2e-hcc-np-convergence-ladder.sh+++ b/scripts/e2e/e2e-hcc-np-convergence-ladder.sh@@ -17,6 +17,10 @@# Anti-vacuity: observe either the unsynced log or the swallowed{unsynced}# metric increment, AND the retry-schedule log. If the window never appears,# retry the restart once. Still unseen → exit 3 (HCC_NP_LADDER_VACUOUS).+# After /ready 200 on that same pod, scrape HTTP /metrics and require+# deferred-unsynced, swallowed{unsynced}, retries, certified, and last-success+# to agree on the Prometheus text. That does NOT attribute certification to+# the 5s timer (watch-recovery may certify first on minikube).set -euo pipefailSCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"CALLER_E2E_KUBECONTEXT="${E2E_KUBECONTEXT:-}"@@ -32,6 +36,9 @@ fisource "${SCRIPT_DIR}/_lib/hcc-watch-recovery-lock.sh"# shellcheck disable=SC1091source "${SCRIPT_DIR}/_lib/hcc-ready-series.sh"+# e2e-lib log() writes stdout. This probe captures helper output as pod names,+# so progress lines must not leak into those substitutions.+log() { echo -e "${CYAN}[E2E]${NC} $*" >&2; }OWNED_MINIKUBE_PROFILE="clerum-fix-447-hcc-certification-watchdog-45626152"VACUOUS_EXIT=3@@ -90,6 +97,7 @@ VACUOUS=0ROLLOUT_TRIGGERED=0OBSERVED_AT=""OBSERVED_POD=""+OBSERVED_POD_UID=""SWALLOW_SOURCE=""READY_AT=""ATTEMPT=0@@ -120,19 +128,79 @@ hcc_pod_metrics() {wget -T 10 -t 1 -qO- http://127.0.0.1:8081/metrics 2>/dev/null || true}-swallowed_unsynced_count() {- local metrics=$1- awk '- $1 ~ /^clerum_hcc_initial_convergence_swallowed_total\{/ &&- $1 ~ /lane="NetworkPolicy"/ &&- $1 ~ /sink="unsynced"/ {- print int($2 + 0)- found = 1+# Label order is not part of the Prometheus contract.+prom_labeled_value() {+ local metrics=$1 name=$2+ shift 2+ local label_spec="" pair+ for pair in "$@"; do+ label_spec="${label_spec}${pair}|"+ done+ awk -v metric_name="$name" -v label_spec="$label_spec" '+ BEGIN {+ n = split(label_spec, required, "|")+ while (n > 0 && required[n] == "") n--+ }+ index($1, metric_name "{") == 1 {+ ok = 1+ for (j = 1; j <= n; j++) {+ if (required[j] == "") continue+ split(required[j], kv, "=")+ needle = kv[1] "=\"" kv[2] "\""+ if (index($1, needle) == 0) ok = 0+ }+ if (ok) {+ print int($NF + 0)+ found = 1+ exit+ }}END { if (!found) print 0 }' <<<"$metrics"}+swallowed_unsynced_count() {+ prom_labeled_value "$1" 'clerum_hcc_initial_convergence_swallowed_total' \+ 'lane=NetworkPolicy' 'sink=unsynced'+}++assert_convergence_metrics_closure() {+ local pod=$1+ local uid_now metrics deferred swallowed retries certified ts+ require_hcc_pod_name "$pod"+ uid_now="$(hcc_pod_uid "$pod")"+ [ -n "$OBSERVED_POD_UID" ] && [ "$uid_now" = "$OBSERVED_POD_UID" ] ||+ die "HCC pod uid changed before /metrics closure (${OBSERVED_POD_UID:-none} -> ${uid_now:-none})"+ metrics="$(hcc_pod_metrics "$pod")"+ [ -n "$metrics" ] || die "HCC /metrics scrape was empty on ${pod}"+ deferred="$(prom_labeled_value "$metrics" 'clerum_hcc_initial_convergence_pass_results_total' \+ 'lane=NetworkPolicy' 'result=deferred-unsynced')"+ swallowed="$(prom_labeled_value "$metrics" 'clerum_hcc_initial_convergence_swallowed_total' \+ 'lane=NetworkPolicy' 'sink=unsynced')"+ retries="$(prom_labeled_value "$metrics" 'clerum_hcc_initial_convergence_retries_total' \+ 'lane=NetworkPolicy')"+ certified="$(prom_labeled_value "$metrics" 'clerum_hcc_initial_convergence_pass_results_total' \+ 'lane=NetworkPolicy' 'result=certified')"+ ts="$(prom_labeled_value "$metrics" 'clerum_hcc_initial_convergence_last_success_timestamp_seconds' \+ 'lane=NetworkPolicy')"+ if [ "${deferred:-0}" -lt 1 ]; then+ fail "INCONCLUSIVE: no deferred-unsynced pass on ${pod} — /metrics closure is vacuous for #447"+ return 1+ fi+ [ "${swallowed:-0}" = "$deferred" ] &&+ ok "wire /metrics: swallowed{unsynced}=${swallowed} == deferred-unsynced=${deferred}" ||+ fail "wire /metrics: swallowed{unsynced}=${swallowed} != deferred-unsynced=${deferred}"+ [ "${retries:-0}" -ge 1 ] &&+ ok "wire /metrics: retries_total=${retries} (armed; not attributed to the 5s timer alone)" ||+ fail "wire /metrics: retries_total=${retries} after a deferred-unsynced pass"+ [ "${certified:-0}" -ge 1 ] &&+ ok "wire /metrics: certified=${certified} on the same pod that deferred" ||+ fail "wire /metrics: certified=${certified} after /ready 200 on a deferred pod"+ [ "$(printf '%s' "$ts" | awk '{ print ($1 > 0) }')" = 1 ] &&+ ok "wire /metrics: last_success_timestamp_seconds>0" ||+ fail "wire /metrics: last_success_timestamp_seconds=${ts} was not set by a certified pass"+}+logs_show_unsynced() {local logs=$1grep -Eq 'caches unsynced|deferred: caches unsynced' <<<"$logs"@@ -147,7 +215,7 @@ write_evidence_artifact() {{echo "=== HCC NP convergence ladder ==="echo "context=${E2E_KUBECONTEXT} attempt=${ATTEMPT} vacuous=${VACUOUS}"- echo "observed_at=${OBSERVED_AT:-none} swallow_source=${SWALLOW_SOURCE:-none} ready_at=${READY_AT:-none}"+ echo "observed_at=${OBSERVED_AT:-none} observed_pod=${OBSERVED_POD:-none} observed_uid=${OBSERVED_POD_UID:-none} swallow_source=${SWALLOW_SOURCE:-none} ready_at=${READY_AT:-none}"echo "=== deployment ==="kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o wide || trueecho "=== pods ==="@@ -211,6 +279,7 @@ observe_unsynced_swallow() {fiif [ "$seen_swallow" = 1 ] && [ "$seen_retry" = 1 ]; thenOBSERVED_AT="$(date +%s)"+ OBSERVED_POD_UID="$(hcc_pod_uid "$pod")"return 0finow="$(date +%s)"@@ -220,6 +289,12 @@ observe_unsynced_swallow() {return 1}+require_hcc_pod_name() {+ local pod=$1+ [[ "$pod" =~ ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ ]] ||+ die "refusing to use a malformed HCC pod name: ${pod}"+}+restart_hcc_and_wait() {local old_pod=$1 old_uid=$2local new_pod new_uid@@ -229,13 +304,13 @@ restart_hcc_and_wait() {kctl rollout status "deployment/${HCC_DEPLOY}" -n "$HCC_NS" --timeout="${ROLLOUT_TIMEOUT_SEC}s" >/dev/null ||die "HCC rollout did not become Ready within ${ROLLOUT_TIMEOUT_SEC}s"new_pod="$(running_hcc_pod)" || true- [ -n "$new_pod" ] || die "no Running HCC pod after rollout"+ require_hcc_pod_name "${new_pod:-}"new_uid="$(hcc_pod_uid "$new_pod")"[ -n "$new_uid" ] || die "could not read replacement pod uid"if [ "$new_pod" = "$old_pod" ] && [ "$new_uid" = "$old_uid" ]; thendie "HCC pod did not change after rollout restart (${old_pod}/${old_uid})"fi- printf '%s\n' "$new_pod"+ NEW_POD="$new_pod"}wait_ready_after_observation() {@@ -278,9 +353,13 @@ old_uid="$(hcc_pod_uid "$old_pod")"swallow_seen=0new_pod=""+NEW_POD=""+require_hcc_pod_name "$old_pod"while [ "$ATTEMPT" -lt "$RESTART_ATTEMPTS" ]; doATTEMPT=$((ATTEMPT + 1))- new_pod="$(restart_hcc_and_wait "$old_pod" "$old_uid")"+ restart_hcc_and_wait "$old_pod" "$old_uid"+ new_pod="$NEW_POD"+ require_hcc_pod_name "$new_pod"if observe_unsynced_swallow "$new_pod"; thenswallow_seen=1ok "observed unsynced swallow (${SWALLOW_SOURCE}) and retry log on ${new_pod}"@@ -299,6 +378,7 @@ fiif wait_ready_after_observation "$new_pod"; thenok "/ready 200 ${READY_AT}s epoch, $((READY_AT - OBSERVED_AT))s after the observed swallow (budget ${READY_BUDGET_SEC}s)"+ assert_convergence_metrics_closure "$new_pod"elsefail "/ready did not become 200 within ${READY_BUDGET_SEC}s after the observed swallow on ${new_pod} (see ${LOG_ARTIFACT})"fi