Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
test(hcc): pin HTTP /ready after an unsynced swallow and fail-closed Minikube probe (#447)
Alfredo Lopeza51de1b945b0
diff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.jsonindex 9783bd956..665d49863 100644--- a/host-context-controller/package-lock.json+++ b/host-context-controller/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-host-context-controller",- "version": "0.1.237",+ "version": "0.1.238","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-host-context-controller",- "version": "0.1.237",+ "version": "0.1.238","license": "SEE LICENSE IN LICENSE","dependencies": {"@clerum/image-policy": "file:../packages/image-policy",diff --git a/host-context-controller/package.json b/host-context-controller/package.jsonindex 63f7ecdd8..68fdd2e0e 100644--- a/host-context-controller/package.json+++ b/host-context-controller/package.json@@ -1,6 +1,6 @@{"name": "clerum-host-context-controller",- "version": "0.1.237",+ "version": "0.1.238","description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API","main": "dist/main.js","engines": {diff --git a/host-context-controller/src/k8sClient.test.ts b/host-context-controller/src/k8sClient.test.tsindex ada7ccaa8..635323bbe 100644--- a/host-context-controller/src/k8sClient.test.ts+++ b/host-context-controller/src/k8sClient.test.ts@@ -3928,11 +3928,23 @@ describe('McpServerWatcher startup', () => {transport: { type: 'streamableHttp' as const, port: 3000 },},})- const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})+ const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})const successTimestampBefore = await readInitialConvergenceMetric('clerum_hcc_initial_convergence_last_success_timestamp_seconds','NetworkPolicy')+ const swallowedBefore = await readLabeledConvergenceMetric(+ 'clerum_hcc_initial_convergence_swallowed_total',+ { lane: 'NetworkPolicy', sink: 'authority-lost' }+ )+ const abortedBefore = await readLabeledConvergenceMetric(+ 'clerum_hcc_initial_convergence_pass_results_total',+ { lane: 'NetworkPolicy', result: 'aborted-authority' }+ )+ const droppedBefore = await readLabeledConvergenceMetric(+ 'clerum_hcc_initial_convergence_effects_dropped_total',+ { lane: 'NetworkPolicy', kind: 'context' }+ )mocks.netPolFullReconcile.mockImplementation(async (contexts, servers, options) => {pass += 1if (pass === 1) {@@ -3968,7 +3980,7 @@ describe('McpServerWatcher startup', () => {expect(appliedContexts).toEqual([])expect(appliedServers).toEqual([])expect(- errorSpy.mock.calls.some(call =>+ warnSpy.mock.calls.some(call =>String(call[0]).includes('pass ended without certifying: inventory authority lost'))).toBe(true)@@ -3985,19 +3997,19 @@ describe('McpServerWatcher startup', () => {lane: 'NetworkPolicy',sink: 'authority-lost',})- ).toBeGreaterThan(0)+ ).toBe(swallowedBefore + 1)expect(await readLabeledConvergenceMetric('clerum_hcc_initial_convergence_pass_results_total', {lane: 'NetworkPolicy',result: 'aborted-authority',})- ).toBeGreaterThan(0)+ ).toBe(abortedBefore + 1)expect(await readLabeledConvergenceMetric('clerum_hcc_initial_convergence_effects_dropped_total', {lane: 'NetworkPolicy',kind: 'context',})- ).toBeGreaterThan(0)+ ).toBe(droppedBefore + 1);(watcher as any).contexts.clear();(watcher as any).servers.clear();(watcher as any).contexts.set('current-context', {@@ -4027,7 +4039,7 @@ describe('McpServerWatcher startup', () => {'NetworkPolicy')).toBeGreaterThan(successTimestampBefore)- errorSpy.mockRestore()+ warnSpy.mockRestore()await watcher.stop()})@@ -4369,6 +4381,14 @@ describe('McpServerWatcher startup', () => {'clerum_hcc_initial_convergence_last_success_timestamp_seconds','NetworkPolicy')+ const swallowedBefore = await readLabeledConvergenceMetric(+ 'clerum_hcc_initial_convergence_swallowed_total',+ { lane: 'NetworkPolicy', sink: 'unsynced' }+ )+ const deferredBefore = await readLabeledConvergenceMetric(+ 'clerum_hcc_initial_convergence_pass_results_total',+ { lane: 'NetworkPolicy', result: 'deferred-unsynced' }+ )await (watcher as any).runInitialNetworkPolicyConvergence()@@ -4381,13 +4401,13 @@ describe('McpServerWatcher startup', () => {lane: 'NetworkPolicy',sink: 'unsynced',})- ).toBeGreaterThan(0)+ ).toBe(swallowedBefore + 1)expect(await readLabeledConvergenceMetric('clerum_hcc_initial_convergence_pass_results_total', {lane: 'NetworkPolicy',result: 'deferred-unsynced',})- ).toBeGreaterThan(0)+ ).toBe(deferredBefore + 1)expect(await readInitialConvergenceMetric('clerum_hcc_initial_convergence_last_success_timestamp_seconds',@@ -4429,6 +4449,10 @@ describe('McpServerWatcher startup', () => {'clerum_hcc_initial_convergence_last_success_timestamp_seconds','NetworkPolicy')+ const certifiedBefore = await readLabeledConvergenceMetric(+ 'clerum_hcc_initial_convergence_pass_results_total',+ { lane: 'NetworkPolicy', result: 'certified' }+ )await (watcher as any).runInitialNetworkPolicyConvergence()@@ -4451,17 +4475,108 @@ describe('McpServerWatcher startup', () => {lane: 'NetworkPolicy',result: 'certified',})- ).toBeGreaterThan(0)+ ).toBe(certifiedBefore + 1)expect(await readInitialConvergenceMetric('clerum_hcc_initial_convergence_last_success_timestamp_seconds','NetworkPolicy')).toBeGreaterThan(successTimestampBefore)++ mocks.netPolFullReconcile.mockClear()+ ;(watcher as any).mcpServerCacheSynced = false+ await (watcher as any).runInitialNetworkPolicyConvergence()+ expect(mocks.netPolFullReconcile).not.toHaveBeenCalled()+ expect((watcher as any).initialConvergenceRetryAttempts.get('NetworkPolicy')).toBe(1)+ await vi.advanceTimersByTimeAsync(5000)+ await flushMicrotasks()+ expect(mocks.netPolFullReconcile).not.toHaveBeenCalled()+ expect((watcher as any).initialConvergenceRetryAttempts.get('NetworkPolicy')).toBe(2)+ expect((watcher as any).initialConvergenceRetryTimers.has('NetworkPolicy')).toBe(true)+ await vi.advanceTimersByTimeAsync(14_999)+ await flushMicrotasks()+ expect(mocks.netPolFullReconcile).not.toHaveBeenCalled()+ ;(watcher as any).mcpServerCacheSynced = true+ await vi.advanceTimersByTimeAsync(1)+ await flushMicrotasks()+ expect(mocks.netPolFullReconcile).toHaveBeenCalledOnce()+ expect((watcher as any).initialConvergenceRetryAttempts.has('NetworkPolicy')).toBe(false)+ expect(+ await readLabeledConvergenceMetric('clerum_hcc_initial_convergence_pass_results_total', {+ lane: 'NetworkPolicy',+ result: 'certified',+ })+ ).toBe(certifiedBefore + 2)warnSpy.mockRestore()await watcher.stop()})+ it('recovers HTTP /ready after a NetworkPolicy unsynced swallow without later watch events', async () => {+ vi.useFakeTimers()+ const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})+ mocks.hasCertifiedSafetyInventory.mockReturnValue(false)+ mocks.netPolFullReconcile.mockImplementation(async (...args: unknown[]) => {+ const options = args[2] as { onAuthoritativeRevocationComplete?: () => void } | undefined+ options?.onAuthoritativeRevocationComplete?.()+ mocks.hasCertifiedSafetyInventory.mockReturnValue(true)+ })+ const watcher = new McpServerWatcher()+ ;(watcher as any).contextCacheSynced = true+ ;(watcher as any).mcpServerCacheSynced = false+ ;(watcher as any).hostCacheSynced = true+ const certifiedBefore = await readLabeledConvergenceMetric(+ 'clerum_hcc_initial_convergence_pass_results_total',+ { lane: 'NetworkPolicy', result: 'certified' }+ )+ const successTimestampBefore = await readInitialConvergenceMetric(+ 'clerum_hcc_initial_convergence_last_success_timestamp_seconds',+ 'NetworkPolicy'+ )+ // Prior tests in this file leave a process-global last-success sample that+ // can sit ahead of a freshly installed fake clock. Pin time past that+ // sample so "timestamp moved" cannot pass by comparing against a rewind.+ vi.setSystemTime(Math.max(Date.now(), Math.round(successTimestampBefore * 1000) + 1000))+ const server = new ContextMapperServer(watcher, 0, undefined, undefined, () =>+ watcher.isReadinessInventoryAuthoritative()+ )+ await server.start()+ server.setReady(true)++ try {+ await (watcher as any).runInitialNetworkPolicyConvergence()+ expect(mocks.netPolFullReconcile).not.toHaveBeenCalled()+ expect((await requestReadyOverHttp(server)).statusCode).toBe(503)+ ;(watcher as any).mcpServerCacheSynced = true+ expect((await requestReadyOverHttp(server)).statusCode).toBe(503)+ await vi.advanceTimersByTimeAsync(5000)+ await flushMicrotasks()++ expect(mocks.netPolFullReconcile).toHaveBeenCalledOnce()+ expect(+ await readLabeledConvergenceMetric('clerum_hcc_initial_convergence_pass_results_total', {+ lane: 'NetworkPolicy',+ result: 'certified',+ })+ ).toBe(certifiedBefore + 1)+ expect(+ await readInitialConvergenceMetric(+ 'clerum_hcc_initial_convergence_last_success_timestamp_seconds',+ 'NetworkPolicy'+ )+ ).toBeGreaterThan(successTimestampBefore)+ expect((await requestReadyOverHttp(server)).statusCode).toBe(200)+ } finally {+ mocks.hasCertifiedSafetyInventory.mockReturnValue(true)+ mocks.netPolFullReconcile.mockImplementation(async (...args: unknown[]) => {+ const options = args[2] as { onAuthoritativeRevocationComplete?: () => void } | undefined+ options?.onAuthoritativeRevocationComplete?.()+ })+ warnSpy.mockRestore()+ await server.stop()+ await watcher.stop()+ }+ })+it('names a generic NetworkPolicy catch failure failed instead of aborted-bump', async () => {const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})const watcher = new McpServerWatcher()diff --git a/host-context-controller/src/k8sClient.ts b/host-context-controller/src/k8sClient.tsindex a1a8e5277..54baa54af 100644--- a/host-context-controller/src/k8sClient.ts+++ b/host-context-controller/src/k8sClient.ts@@ -3156,7 +3156,7 @@ export class McpServerWatcher implements McpServerProvider {},})if (!contextInventoryAuthoritative() || !serverInventoryAuthoritative()) {- console.error('[K8s] pass ended without certifying: inventory authority lost')+ console.warn('[K8s] pass ended without certifying: inventory authority lost')initialConvergenceSwallowedTotal.inc({ lane: 'NetworkPolicy', sink: 'authority-lost' })observeInitialNetworkPolicyPass(startedAtMs, 'aborted-authority')this.scheduleInitialConvergenceRetry('NetworkPolicy')diff --git a/scripts/e2e/e2e-hcc-np-convergence-ladder.sh b/scripts/e2e/e2e-hcc-np-convergence-ladder.shnew file mode 100755index 000000000..159cfe056--- /dev/null+++ b/scripts/e2e/e2e-hcc-np-convergence-ladder.sh@@ -0,0 +1,304 @@+#!/usr/bin/env bash+# HCC NetworkPolicy initial-convergence ladder after a swallowed unsynced pass.+#+# SCOPE — this probe proves ONLY that Change A can fire on a real HCC process+# in the owned Minikube profile: a rolling restart may hit caches-unsynced,+# the retry ladder re-arms, and /ready becomes 200 within 300s AFTER that+# swallow is observed. It does NOT reproduce the GKE 17–35 min watch-recycle+# livelock. Minikube often syncs both inventories before the first NP pass;+# a lucky /ready 200 without a swallow is FAIL (vacuous), not green. The RED+# for the silent-return bug lives in host-context-controller/src/k8sClient.test.ts.+#+# Action: kubectl rollout restart of host-context-controller. Do not change+# strategy, replicas, or PVCs. Do not Recreate the Deployment object. Do not+# switch the global current-context. Do not adopt another profile's port-forward;+# /ready and /metrics are probed with kubectl --context exec wget on :8081.+#+# Anti-vacuity: observe either the unsynced log or the swallowed{unsynced}+# metric increment, AND the retry-schedule log. If the window never appears,+# retry the restart once. Still unseen → exit 3 (HCC_NP_LADDER_VACUOUS).+set -euo pipefail+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"+CALLER_E2E_KUBECONTEXT="${E2E_KUBECONTEXT:-}"+# shellcheck source=scripts/e2e/e2e-lib.sh+source "${SCRIPT_DIR}/e2e-lib.sh"+# e2e-lib maps KUBECONTEXT / E2E_K8S_CONTEXT onto E2E_KUBECONTEXT and+# clobbers a pre-set value. Keep an explicit caller context when those+# aliases were empty.+if [ -z "${KUBECONTEXT:-}" ] && [ -z "${E2E_K8S_CONTEXT:-}" ] && [ -n "$CALLER_E2E_KUBECONTEXT" ]; then+ E2E_KUBECONTEXT="$CALLER_E2E_KUBECONTEXT"+fi+# shellcheck disable=SC1091+source "${SCRIPT_DIR}/_lib/hcc-watch-recovery-lock.sh"+# shellcheck disable=SC1091+source "${SCRIPT_DIR}/_lib/hcc-ready-series.sh"++OWNED_MINIKUBE_PROFILE="clerum-fix-447-hcc-certification-watchdog-45626152"+VACUOUS_EXIT=3++[ -n "$E2E_KUBECONTEXT" ] || {+ echo "KUBECONTEXT/E2E_K8S_CONTEXT must select the owned branch-scoped minikube context." >&2+ exit 1+}+is_branch_scoped_e2e_context "$E2E_KUBECONTEXT" || {+ echo "Refusing NP ladder probe on non-branch context '${E2E_KUBECONTEXT}'." >&2+ exit 1+}+[ "$E2E_KUBECONTEXT" = "$OWNED_MINIKUBE_PROFILE" ] || {+ echo "Refusing NP ladder probe: this script targets only ${OWNED_MINIKUBE_PROFILE}," >&2+ echo "got '${E2E_KUBECONTEXT}'. Do not point it at another worktree's profile." >&2+ exit 1+}+[ "${MINIKUBE_PROFILE:-}" = "$E2E_KUBECONTEXT" ] || {+ echo "MINIKUBE_PROFILE must explicitly select ${E2E_KUBECONTEXT}, got ${MINIKUBE_PROFILE:-missing}." >&2+ exit 1+}+require_safe_kube_context+command -v jq >/dev/null 2>&1 || {+ echo "jq is required" >&2+ exit 1+}+[ "${E2E_HCC_NP_LADDER_FAULT_INJECTION:-0}" = 1 ] || {+ echo "Set E2E_HCC_NP_LADDER_FAULT_INJECTION=1 to acknowledge a rolling restart of HCC." >&2+ exit 1+}+[ "${EXPECT_LIVELOCK:-0}" != 1 ] || {+ echo "EXPECT_LIVELOCK=1 is not supported: this probe does not reproduce the GKE" >&2+ echo "watch-recycle livelock. The RED lives in host-context-controller/src/k8sClient.test.ts." >&2+ exit 1+}+kctl get nodes -o json | jq -e --arg c "$E2E_KUBECONTEXT" \+ 'any(.items[]; .metadata.labels["minikube.k8s.io/name"] == $c)' >/dev/null ||+ {+ echo "Refusing NP ladder probe: target is not this profile's minikube node." >&2+ exit 1+ }++HCC_NS="${HCC_NS:-control-plane}"+HCC_DEPLOY="${HCC_DEPLOY:-host-context-controller}"+SWALLOW_OBSERVE_SEC="${SWALLOW_OBSERVE_SEC:-90}"+READY_BUDGET_SEC="${READY_BUDGET_SEC:-300}"+ROLLOUT_TIMEOUT_SEC="${ROLLOUT_TIMEOUT_SEC:-180}"+RESTART_ATTEMPTS="${RESTART_ATTEMPTS:-2}"++RUN_ID="$(date +%s)-$$"+LOG_ARTIFACT="$(mktemp "${TMPDIR:-/tmp}/hcc-np-convergence-ladder.XXXXXX")"+READY_SERIES="$(mktemp "${TMPDIR:-/tmp}/hcc-np-convergence-ladder-series.XXXXXX")"+# shellcheck disable=SC2034+HCC_GATE_LOCK_ACQUIRED=0 HCC_GATE_LOCK_NAME="" HCC_GATE_LOCK_UID="" HCC_GATE_FINALIZATION_FAILURE=""+VACUOUS=0+ROLLOUT_TRIGGERED=0+OBSERVED_AT=""+OBSERVED_POD=""+SWALLOW_SOURCE=""+READY_AT=""+ATTEMPT=0++die() {+ fail "$*"+ exit 1+}++hcc_ready_now() {+ local pod+ pod="$(running_hcc_pod)" && [ -n "$pod" ] && [ "$(ready_status "$pod")" = 200 ]+}++hcc_pod_uid() {+ local pod=$1+ kctl get pod "$pod" -n "$HCC_NS" -o jsonpath='{.metadata.uid}'+}++hcc_pod_logs() {+ local pod=$1+ kctl logs "pod/$pod" -n "$HCC_NS" -c host-context-controller 2>/dev/null || true+}++hcc_pod_metrics() {+ local pod=$1+ kctl exec "pod/$pod" -n "$HCC_NS" -c host-context-controller -- \+ wget -T 10 -t 1 -qO- http://127.0.0.1:8081/metrics 2>/dev/null || true+}++swallowed_unsynced_count() {+ local metrics=$1+ awk '+ $1 ~ /^clerum_hcc_initial_convergence_swallowed_total\{/ &&+ $1 ~ /lane="NetworkPolicy"/ &&+ $1 ~ /sink="unsynced"/ {+ print int($2 + 0)+ found = 1+ }+ END { if (!found) print 0 }+ ' <<<"$metrics"+}++logs_show_unsynced() {+ local logs=$1+ grep -Eq 'caches unsynced|deferred: caches unsynced' <<<"$logs"+}++logs_show_retry() {+ local logs=$1+ grep -Fq 'Scheduling initial NetworkPolicy background convergence retry' <<<"$logs"+}++write_evidence_artifact() {+ {+ echo "=== HCC NP convergence ladder ==="+ echo "context=${E2E_KUBECONTEXT} attempt=${ATTEMPT} vacuous=${VACUOUS}"+ echo "observed_at=${OBSERVED_AT:-none} swallow_source=${SWALLOW_SOURCE:-none} ready_at=${READY_AT:-none}"+ echo "=== deployment ==="+ kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o wide || true+ echo "=== pods ==="+ kctl get pods -n "$HCC_NS" -l "app=${HCC_DEPLOY}" -o wide || true+ if [ -n "${OBSERVED_POD:-}" ]; then+ echo "=== logs ${OBSERVED_POD} ==="+ hcc_pod_logs "$OBSERVED_POD" || true+ echo "=== metrics ${OBSERVED_POD} ==="+ hcc_pod_metrics "$OBSERVED_POD" || true+ fi+ echo "=== /ready series ==="+ cat "$READY_SERIES" || true+ } >"$LOG_ARTIFACT" 2>&1 || true+}++cleanup() {+ local status=$? cleanup_failed=0 restore_ok=1+ trap - EXIT+ set +e+ if [ "$ROLLOUT_TRIGGERED" = 1 ]; then+ kctl rollout status deployment "$HCC_DEPLOY" -n "$HCC_NS" --timeout=240s >/dev/null 2>&1 || restore_ok=0+ fi+ [ "$restore_ok" = 1 ] || cleanup_failed=1+ finalize_hcc_watch_gate_lock "$cleanup_failed" "$restore_ok" || cleanup_failed=1+ write_evidence_artifact+ log "Evidence artifact: ${LOG_ARTIFACT}"+ print_results || status=1+ [ "$cleanup_failed" = 0 ] || status=1+ rm -f "$READY_SERIES"+ if [ "$VACUOUS" = 1 ]; then+ echo "HCC_NP_LADDER_VACUOUS: unsynced swallow never observed after ${RESTART_ATTEMPTS} restart(s)." >&2+ exit "$VACUOUS_EXIT"+ fi+ exit "$status"+}++# Observe Change A on the replacement pod: unsynced swallow (log or metric)+# plus the retry-arming log. A 200 without this window is not evidence.+observe_unsynced_swallow() {+ local pod=$1+ local deadline now logs metrics swallowed+ local seen_swallow=0 seen_retry=0+ deadline=$(($(date +%s) + SWALLOW_OBSERVE_SEC))+ OBSERVED_POD="$pod"+ SWALLOW_SOURCE=""+ while :; do+ logs="$(hcc_pod_logs "$pod")"+ metrics="$(hcc_pod_metrics "$pod")"+ swallowed="$(swallowed_unsynced_count "$metrics")"+ if [ "$seen_swallow" = 0 ]; then+ if logs_show_unsynced "$logs"; then+ seen_swallow=1+ SWALLOW_SOURCE="log"+ elif [ "${swallowed:-0}" -gt 0 ]; then+ seen_swallow=1+ SWALLOW_SOURCE="metric"+ fi+ fi+ if [ "$seen_retry" = 0 ] && logs_show_retry "$logs"; then+ seen_retry=1+ fi+ if [ "$seen_swallow" = 1 ] && [ "$seen_retry" = 1 ]; then+ OBSERVED_AT="$(date +%s)"+ return 0+ fi+ now="$(date +%s)"+ [ "$now" -lt "$deadline" ] || break+ sleep 1+ done+ return 1+}++restart_hcc_and_wait() {+ local old_pod=$1 old_uid=$2+ local new_pod new_uid+ log "Rolling restart of ${HCC_NS}/${HCC_DEPLOY} (attempt ${ATTEMPT})"+ kctl rollout restart "deployment/${HCC_DEPLOY}" -n "$HCC_NS" >/dev/null+ ROLLOUT_TRIGGERED=1+ kctl rollout status "deployment/${HCC_DEPLOY}" -n "$HCC_NS" --timeout="${ROLLOUT_TIMEOUT_SEC}s" >/dev/null ||+ die "HCC rollout did not become Ready within ${ROLLOUT_TIMEOUT_SEC}s"+ new_pod="$(running_hcc_pod)" || true+ [ -n "$new_pod" ] || die "no Running HCC pod after rollout"+ new_uid="$(hcc_pod_uid "$new_pod")"+ [ -n "$new_uid" ] || die "could not read replacement pod uid"+ if [ "$new_pod" = "$old_pod" ] && [ "$new_uid" = "$old_uid" ]; then+ die "HCC pod did not change after rollout restart (${old_pod}/${old_uid})"+ fi+ printf '%s\n' "$new_pod"+}++wait_ready_after_observation() {+ local pod=$1+ local deadline now status+ deadline=$((OBSERVED_AT + READY_BUDGET_SEC))+ while :; do+ now="$(date +%s)"+ status="$(ready_status "$pod")"+ printf '%s %s after-swallow\n' "$now" "$status" >>"$READY_SERIES"+ if [ "$status" = 200 ]; then+ READY_AT="$now"+ return 0+ fi+ [ "$now" -lt "$deadline" ] || break+ sleep 1+ done+ return 1+}++trap cleanup EXIT++acquire_hcc_watch_gate_lock++replicas="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o jsonpath='{.spec.replicas}')"+[ "$replicas" = 1 ] || die "expected exactly one HCC replica, found ${replicas:-unknown}"+ready_replicas="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o jsonpath='{.status.readyReplicas}')"+[ "$ready_replicas" = 1 ] || die "HCC Deployment is not Ready (readyReplicas=${ready_replicas:-unknown})"+strategy_type="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o jsonpath='{.spec.strategy.type}')"+[ "$strategy_type" = Recreate ] ||+ die "refusing to mutate a non-Recreate HCC (found '${strategy_type:-unset}'); this probe must not change strategy"++header "HCC NP convergence ladder (owned Minikube only)"+log "This probe does not claim to reproduce GKE watch-recycle. Vacuous swallow = fail-closed."++old_pod="$(running_hcc_pod)" || true+[ -n "$old_pod" ] || die "no Running HCC pod before restart"+old_uid="$(hcc_pod_uid "$old_pod")"+[ -n "$old_uid" ] || die "could not pin the pre-restart pod uid"++swallow_seen=0+new_pod=""+while [ "$ATTEMPT" -lt "$RESTART_ATTEMPTS" ]; do+ ATTEMPT=$((ATTEMPT + 1))+ new_pod="$(restart_hcc_and_wait "$old_pod" "$old_uid")"+ if observe_unsynced_swallow "$new_pod"; then+ swallow_seen=1+ ok "observed unsynced swallow (${SWALLOW_SOURCE}) and retry log on ${new_pod}"+ break+ fi+ log "unsynced swallow not observed on ${new_pod} within ${SWALLOW_OBSERVE_SEC}s"+ old_pod="$new_pod"+ old_uid="$(hcc_pod_uid "$new_pod")"+done++if [ "$swallow_seen" != 1 ]; then+ VACUOUS=1+ fail "VACUOUS: never saw caches-unsynced swallow or swallowed{unsynced} increment plus retry log after ${RESTART_ATTEMPTS} restart(s) — a lucky /ready 200 is not a pass (see ${LOG_ARTIFACT})"+ exit "$VACUOUS_EXIT"+fi++if wait_ready_after_observation "$new_pod"; then+ ok "/ready 200 ${READY_AT}s epoch, $((READY_AT - OBSERVED_AT))s after the observed swallow (budget ${READY_BUDGET_SEC}s)"+else+ fail "/ready did not become 200 within ${READY_BUDGET_SEC}s after the observed swallow on ${new_pod} (see ${LOG_ARTIFACT})"+fi