Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

test(hcc): verify omissions only after settled isolated recovery

Alfredo Lopeze1ee1e083d26
diff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.json
index 7b01d0815..049ea56b4 100644
--- a/host-context-controller/package-lock.json
+++ b/host-context-controller/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.315",
+ "version": "0.1.316",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-host-context-controller",
- "version": "0.1.315",
+ "version": "0.1.316",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@clerum/codex-catalog-projection": "file:../packages/codex-catalog-projection",
diff --git a/host-context-controller/package.json b/host-context-controller/package.json
index ca8fc377d..ad1265e2a 100644
--- a/host-context-controller/package.json
+++ b/host-context-controller/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.315",
+ "version": "0.1.316",
"description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API",
"main": "dist/main.js",
"engines": {
diff --git a/host-context-controller/src/k8sClient.policyEligibility.integration.test.ts b/host-context-controller/src/k8sClient.policyEligibility.integration.test.ts
index 947a53c4f..5c744aced 100644
--- a/host-context-controller/src/k8sClient.policyEligibility.integration.test.ts
+++ b/host-context-controller/src/k8sClient.policyEligibility.integration.test.ts
@@ -880,4 +880,88 @@ describe('WATCH and recovered LIST policy effects', () => {
await drain(state)
expect(fixture.policies.get(desired[0])?.spec).toEqual(desired[1].spec)
})
+ it('A-T6 recovery scheduling: simultaneous retirement defers MCP until Context restores authority', async () => {
+ const state = await settled()
+ const before = structuredClone([...fixture.policies.entries()])
+ expect(before).toHaveLength(17)
+ const log = vi.spyOn(hccLogger, 'info')
+ const originalList = fixture.list.getMockImplementation()!
+ for (let cycle = 0; cycle < 3; cycle++) {
+ await vi.advanceTimersByTimeAsync(7000)
+ let release!: () => void, entered!: () => void
+ const hold = new Promise<void>(resolve => {
+ release = resolve
+ })
+ const started = new Promise<void>(resolve => {
+ entered = resolve
+ })
+ fixture.list.mockImplementation(async args => {
+ if (args.plural === 'contexts') {
+ entered()
+ await hold
+ }
+ return originalList(args)
+ })
+ ;(state as any).retireContextWatch()
+ ;(state as any).retireMcpServerWatch()
+ const contextRecovery = state.recoverContextInventoryAndWatch()
+ await started
+ expect(await state.recoverMcpServerInventoryAndWatch()).toBe(true)
+ expect(state.contextCacheSynced).toBe(false)
+ release()
+ expect(await contextRecovery).toBe(true)
+ await drain(state)
+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(true)
+ expect([...fixture.policies.entries()]).toEqual(before)
+ }
+ const decisions = log.mock.calls
+ .map(([, data]) => data as any)
+ .filter(data => data?.event === 'networkpolicy-recovery-decision')
+ const mcps = decisions.filter(data => data.kind === 'McpServer')
+ const contexts = decisions.filter(data => data.kind === 'Context')
+ expect(mcps).toHaveLength(3)
+ expect(mcps.every(data => data.decision === 'defer' && data.reason === 'no-authority')).toBe(
+ true
+ )
+ expect(contexts).toHaveLength(3)
+ expect(
+ contexts.every(data => data.decision === 'skip' && data.reason === 'identical-complete')
+ ).toBe(true)
+ })
+
+ it('A-T6 recovery scheduling: independent recoveries skip while their peer stays authoritative', async () => {
+ const state = await settled()
+ const before = structuredClone([...fixture.policies.entries()])
+ expect(before).toHaveLength(17)
+ const log = vi.spyOn(hccLogger, 'info')
+ const admission = vi.spyOn(state, 'runInitialConvergence')
+ for (let cycle = 0; cycle < 3; cycle++) {
+ await vi.advanceTimersByTimeAsync(7000)
+ expect(state.contextCacheSynced).toBe(true)
+ ;(state as any).retireMcpServerWatch()
+ expect(await state.recoverMcpServerInventoryAndWatch()).toBe(true)
+ await drain(state)
+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(true)
+ expect([...fixture.policies.entries()]).toEqual(before)
+ expect(state.mcpServerCacheSynced).toBe(true)
+ ;(state as any).retireContextWatch()
+ expect(await state.recoverContextInventoryAndWatch()).toBe(true)
+ await drain(state)
+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(true)
+ expect([...fixture.policies.entries()]).toEqual(before)
+ }
+ const decisions = log.mock.calls
+ .map(([, data]) => data as any)
+ .filter(data => data?.event === 'networkpolicy-recovery-decision')
+ const mcpSkips = decisions.filter(
+ data => data.kind === 'McpServer' && data.decision === 'skip'
+ ).length
+ const contextSkips = decisions.filter(
+ data => data.kind === 'Context' && data.decision === 'skip'
+ ).length
+ const npAdmissions = admission.mock.calls.filter(([lane]) => lane === 'NetworkPolicy').length
+ expect(mcpSkips).toBe(3)
+ expect(contextSkips).toBe(3)
+ expect(npAdmissions).toBe(0)
+ })
})
diff --git a/scripts/e2e/_lib/hcc-networkpolicy-lifecycle.sh b/scripts/e2e/_lib/hcc-networkpolicy-lifecycle.sh
index c794dbd6f..0dee864d9 100755
--- a/scripts/e2e/_lib/hcc-networkpolicy-lifecycle.sh
+++ b/scripts/e2e/_lib/hcc-networkpolicy-lifecycle.sh
@@ -145,8 +145,9 @@ np604_before_observation() {
"e2e.clerum.io/observer=${RUN_ID}" --overwrite >/dev/null
wait_until 15 'NP604 policy observer positive witness' np604_observer_witness || die 'NP604 observer is not live'
if [ "${E2E_HCC_PR_A:-0}" = 1 ]; then
- HCC_PR_A_MCP_SKIP_BASE="$(hcc_pr_a_decision_count McpServer skip)"
- HCC_PR_A_CONTEXT_SKIP_BASE="$(hcc_pr_a_decision_count Context skip)"
+ # Consumed by the checkpoint helper in hcc-watch-pr-a.sh.
+ # shellcheck disable=SC2034
+ HCC_PR_A_OBSERVATION_LOG_LINE_BASE="$(wc -l < "$HCC_LOG_BUFFER" | tr -d ' ')"
fi
NP604_CUT_BASE="$(count_buffer '\[K8s\] Context watch ended; recovering authoritative inventory')"
}
@@ -154,9 +155,7 @@ np604_before_observation() {
np604_after_observation() {
np604_observer_alive || die 'NP604 observer stopped before completing observation'
if [ "${E2E_HCC_PR_A:-0}" = 1 ]; then
- [ "$(( $(hcc_pr_a_decision_count McpServer skip) - HCC_PR_A_MCP_SKIP_BASE ))" -ge 3 ] || die 'PR A requires 3 completed identical MCP omissions'
- [ "$(( $(hcc_pr_a_decision_count Context skip) - HCC_PR_A_CONTEXT_SKIP_BASE ))" -ge 3 ] || die 'PR A requires 3 completed identical Context omissions'
- printf 'PR_A_IDENTICAL_RECOVERIES=PASS (at least 3 per watch)\n'
+ hcc_pr_a_recovery_checkpoint
fi
local current_cuts
current_cuts="$(count_buffer '\[K8s\] Context watch ended; recovering authoritative inventory')"
diff --git a/scripts/e2e/_lib/hcc-watch-lifecycle-cleanup.sh b/scripts/e2e/_lib/hcc-watch-lifecycle-cleanup.sh
index 33383fdf2..f075eea36 100755
--- a/scripts/e2e/_lib/hcc-watch-lifecycle-cleanup.sh
+++ b/scripts/e2e/_lib/hcc-watch-lifecycle-cleanup.sh
@@ -152,6 +152,14 @@ cleanup_hcc_lifecycle() (
fi
HCC_CLEANUP_PHASE_DEADLINE=$((started + 270))
+ if { [ "$status" != 0 ] || [ "$cleanup_failed" != 0 ]; } &&
+ [ "${E2E_HCC_PR_A:-0}" = 1 ] && [ "${NP604_CREATED:-0}" = 1 ] &&
+ [ -d "${NP604_EVIDENCE:-}" ] && [ -r "$HCC_LOG_BUFFER" ]; then
+ # Keep structured evidence for later phases too, before deleting the raw
+ # buffer. Diagnostics do not replace or change the existing failure verdict.
+ hcc_pr_a_recovery_checkpoint 0 "$NP604_EVIDENCE/failure-recovery-observation.jsonl" ||
+ printf 'PR_A_FAILURE_CHECKPOINT=UNAVAILABLE\n' >&2
+ fi
finalize_hcc_watch_gate_lock "$cleanup_failed" "$restore_ok" || cleanup_failed=1
print_results || cleanup_failed=1
if [ "$status" -eq 0 ] && [ "$cleanup_failed" != 0 ]; then status=1; fi
diff --git a/scripts/e2e/_lib/hcc-watch-pr-a.sh b/scripts/e2e/_lib/hcc-watch-pr-a.sh
index 42d4ca858..7ba1585a5 100644
--- a/scripts/e2e/_lib/hcc-watch-pr-a.sh
+++ b/scripts/e2e/_lib/hcc-watch-pr-a.sh
@@ -45,6 +45,80 @@ hcc_pr_a_logs() {
jq -Rrc 'fromjson? | select(.event=="networkpolicy-recovery-decision" or .event=="networkpolicy-request" or .event=="networkpolicy-pass-start" or .event=="networkpolicy-pass-result")' "$HCC_LOG_BUFFER"
}
+hcc_pr_a_recovery_checkpoint() {
+ # Defaults preserve the simultaneous-stress diagnostic checkpoint.
+ local line_base=${1:-$HCC_PR_A_OBSERVATION_LOG_LINE_BASE}
+ local destination=${2:-$NP604_EVIDENCE/recovery-observation.jsonl}
+ # Persist only structured observation fields before a failing assertion can
+ # enter cleanup and remove the original log buffer. Never persist payloads.
+ tail -n "+$((line_base + 1))" "$HCC_LOG_BUFFER" |
+ jq -Rrc 'fromjson? | select(.event=="networkpolicy-recovery-decision" or
+ .event=="networkpolicy-request" or .event=="networkpolicy-pass-start" or .event=="networkpolicy-pass-result") |
+ {ts,event,kind,decision,reason,contextRevision,serverRevision,passId,causes,trailing,result,cause,admission} |
+ with_entries(select(.value!=null))' > "$destination"
+ printf 'PR_A_RECOVERY_DECISION_COUNTS (observation window)\n'
+ jq -sc '[.[]|select(.event=="networkpolicy-recovery-decision")] |
+ group_by([.kind,.decision,.reason]) | map({kind:.[0].kind,decision:.[0].decision,reason:.[0].reason,count:length})' \
+ "$destination"
+}
+
+hcc_pr_a_identical_skip_after() {
+ local line_base=$1 kind=$2 revision=$3
+ require_hcc_recovery_log_stream || return 1
+ tail -n "+$((line_base + 1))" "$HCC_LOG_BUFFER" | jq -Rse --arg kind "$kind" --argjson revision "$revision" '
+ split("\n") | map(fromjson?) | any(.event=="networkpolicy-recovery-decision" and .kind==$kind and
+ .decision=="skip" and .reason=="identical-complete" and
+ .contextRevision==$revision.contextRevision and .serverRevision==$revision.serverRevision)' >/dev/null
+}
+
+hcc_pr_a_no_recovery_admission_after() {
+ tail -n "+$(($1 + 1))" "$HCC_LOG_BUFFER" | jq -Rse '
+ split("\n") | map(fromjson?) | all(.[];
+ ((.event=="networkpolicy-request" and (.cause=="mcp-recovery" or .cause=="context-recovery")) or
+ (.event=="networkpolicy-pass-start" and any(.causes[]?; .=="mcp-recovery" or .=="context-recovery"))) | not)' >/dev/null
+}
+
+hcc_pr_a_controlled_omissions_core() {
+ local line_base=$1 kind cycle before revision periodic affected control
+ require_hcc_recovery_log_stream || return 1
+ hcc_pr_a_gate 200 || return 1
+ revision="$(hcc_pr_a_logs | jq -sce '[.[]|select(.event=="networkpolicy-pass-result" and .result=="certified")][-1] |
+ {contextRevision,serverRevision} | select(.contextRevision!=null and .serverRevision!=null)')" || return 1
+ periodic="$(hcc_pr_a_periodic_started_count)" || return 1
+ affected="$(np604_snapshot "$NP604_SERVER")" || return 1
+ control="$(np604_snapshot "$NP604_CONTROL")" || return 1
+ [ "$(jq length <<< "$affected")" = 4 ] && [ "$(jq length <<< "$control")" = 4 ] || return 1
+ for kind in McpServer Context; do
+ for cycle in 1 2 3; do
+ before="$(wc -l < "$HCC_LOG_BUFFER" | tr -d ' ')" || return 1
+ hcc_pr_a_cut "$kind" || return 1
+ wait_until 8 "fresh identical-complete ${kind} omission ${cycle}" \
+ hcc_pr_a_identical_skip_after "$before" "$kind" "$revision" || return 1
+ wait_until 5 'protected API recovered after controlled cut' hcc_pr_a_gate 200 || return 1
+ hcc_pr_a_no_recovery_admission_after "$line_base" || return 1
+ [ "$(hcc_pr_a_periodic_started_count)" = "$periodic" ] || {
+ printf 'PR A controlled omission phase overlapped a periodic pass\n' >&2; return 1;
+ }
+ done
+ done
+ np604_invoke "$NP604_SERVER" || return 1
+ np604_invoke "$NP604_CONTROL" || return 1
+ [ "$(np604_snapshot "$NP604_SERVER")" = "$affected" ] || return 1
+ [ "$(np604_snapshot "$NP604_CONTROL")" = "$control" ] || return 1
+ [ "$(hcc_pr_a_periodic_started_count)" = "$periodic" ] || return 1
+ require_hcc_recovery_log_stream || return 1
+ hcc_pr_a_no_recovery_admission_after "$line_base"
+}
+
+hcc_pr_a_controlled_omissions() {
+ local line_base status=0
+ line_base="$(wc -l < "$HCC_LOG_BUFFER" | tr -d ' ')" || return 1
+ hcc_pr_a_controlled_omissions_core "$line_base" || status=$?
+ hcc_pr_a_recovery_checkpoint "$line_base" "$NP604_EVIDENCE/controlled-recovery-observation.jsonl"
+ [ "$status" = 0 ] || die 'PR A controlled identical recoveries failed; checkpoint retained'
+ printf 'PR_A_IDENTICAL_RECOVERIES=PASS (3 sequential cuts per watch; no recovery admission)\n'
+}
+
hcc_pr_a_decision_after() {
local baseline=$1 kind=$2 decision=$3
require_hcc_recovery_log_stream || return 1
@@ -87,6 +161,10 @@ hcc_pr_a_run() {
periodic="$(hcc_pr_a_periodic_count)"
wait_until 75 'PR A real periodic completion' hcc_pr_a_periodic_after "$periodic" || die 'PR A periodic path inactive'
wait_until 15 'PR A periodic runtime effects complete' hcc_pr_a_runtime_completed || die 'PR A runtime did not complete before drift'
+ # The B stress window may legitimately retain repair work. Test omission
+ # separately after healthy binding, periodic policy/runtime completion and
+ # the driver's acknowledged pause of automatic churn; never warm up on skips.
+ hcc_pr_a_controlled_omissions
periodic="$(hcc_pr_a_periodic_count)"
context_spec="$(np604_kctl get mcpserver "$NP604_SERVER" -n "$MCP_NS" -o json | jq -Sc '{uid:.metadata.uid,generation:.metadata.generation,spec}')"
hcc_pr_a_remove_service
diff --git a/scripts/tests/test-hcc-watch-pr-a.sh b/scripts/tests/test-hcc-watch-pr-a.sh
index bfdb71335..31bcecf65 100644
--- a/scripts/tests/test-hcc-watch-pr-a.sh
+++ b/scripts/tests/test-hcc-watch-pr-a.sh
@@ -18,6 +18,34 @@ if hcc_pr_a_decision_after 0 Context skip; then echo 'FAIL: deferral counted as
if hcc_pr_a_decision_after 0 McpServer skip; then echo 'FAIL: empty observations passed'; exit 1; fi
printf 'PASS: recovery observation needs a new explicit omission of the exact kind\n'
+# The unchanged real-cut stress assertion fails after a safe checkpoint.
+# Deleting its raw buffer afterwards models the existing EXIT cleanup.
+mkdir -p "$tmp/checkpoint"
+printf '%s\n' 'before-observation' \
+ '{"event":"networkpolicy-recovery-decision","kind":"McpServer","decision":"request","reason":"work-pending","contextRevision":2,"serverRevision":3,"body":"synthetic-private-payload"}' \
+ '{"event":"networkpolicy-pass-start","passId":4,"causes":["mcp-recovery"],"contextRevision":2,"serverRevision":3,"headers":"synthetic-private-payload"}' > "$tmp/failed-runtime-buffer"
+if (
+ source "$ROOT/scripts/e2e/_lib/hcc-networkpolicy-lifecycle.sh"
+ E2E_HCC_PR_A=1 NP604_CUT_BASE=0
+ HCC_PR_A_OBSERVATION_LOG_LINE_BASE=1 HCC_LOG_BUFFER="$tmp/failed-runtime-buffer"
+ NP604_EVIDENCE="$tmp/checkpoint"
+ np604_observer_alive() { :; }
+ count_buffer() { printf 0; }
+ die() { printf '%s\n' "$*" >&2; exit 1; }
+ np604_after_observation
+) > "$tmp/checkpoint-result" 2>&1; then
+ echo 'FAIL: zero real stress cuts unexpectedly passed' >&2; exit 1
+fi
+rm "$tmp/failed-runtime-buffer"
+jq -se 'length==2 and .[0].reason=="work-pending" and .[1].passId==4 and
+ all(.[]; has("body")|not) and all(.[]; has("headers")|not)' "$tmp/checkpoint/recovery-observation.jsonl" >/dev/null
+grep -q 'NP604 needs three complete reconnect cycles' "$tmp/checkpoint-result"
+grep -q '"reason":"work-pending","count":1' "$tmp/checkpoint-result"
+if grep -q 'synthetic-private-payload' "$tmp/checkpoint-result" "$tmp/checkpoint/recovery-observation.jsonl"; then
+ echo 'FAIL: payload leaked into checkpoint' >&2; exit 1
+fi
+printf 'PASS: unchanged stress assertion retains a projected checkpoint and grouped reasons after raw-buffer cleanup\n'
+
NP604_SERVER=fixture-affected MCP_NS=mcp-server HCC_PR_A_SERVICE_UID=old
np604_kctl() { cat "$tmp/service"; }
for variation in new old wrong-label wrong-port; do
@@ -43,6 +71,7 @@ printf 'PASS: actual absence and new correctly scoped Service are required\n'
(
NP604_CONTROL=fixture-control
hcc_pr_a_gate() { :; }
+ hcc_pr_a_controlled_omissions() { :; }
np604_snapshot() { printf '[]'; }
hcc_pr_a_periodic_count() { printf 1; }
hcc_pr_a_periodic_started_count() { printf 1; }
@@ -174,3 +203,57 @@ for mode in 0 1; do
done
done
printf 'PASS: PR A retries within budget; legacy callers without wait_until remain strict and one-shot\n'
+
+# Controlled omission has its own positive denominator: six real cut entries,
+# then fresh same-kind/same-revision identical-complete decisions. Exercise
+# the real phase body and checkpoint against hostile boundary outcomes.
+for variant in success wrong-kind replay wrong-reason wrong-revision admission trailing mutation periodic; do
+ mkdir -p "$tmp/controlled-$variant"
+ printf '%s\n' '{"event":"networkpolicy-pass-result","result":"certified","contextRevision":2,"serverRevision":3,"causes":["startup"]}' > "$tmp/controlled-$variant/buffer"
+ status=0
+ (
+ HCC_LOG_BUFFER="$tmp/controlled-$variant/buffer" NP604_EVIDENCE="$tmp/controlled-$variant"
+ NP604_SERVER=fixture-affected NP604_CONTROL=fixture-control cuts=0
+ require_hcc_recovery_log_stream() { :; }
+ hcc_pr_a_gate() { :; }
+ np604_invoke() { :; }
+ np604_snapshot() {
+ if [[ "$variant" = mutation && "$cuts" -gt 0 ]]; then
+ printf '[{"name":"changed"},{},{},{}]'
+ else
+ printf '[{"name":"original"},{},{},{}]'
+ fi
+ }
+ wait_until() { shift 2; "$@"; }
+ hcc_pr_a_cut() {
+ cuts=$((cuts + 1))
+ local kind=$1 reason=identical-complete revision=2
+ [[ "$variant" != replay ]] || return 0
+ [[ "$variant" != wrong-kind ]] || kind=other
+ [[ "$variant" != wrong-reason ]] || reason=work-pending
+ [[ "$variant" != wrong-revision ]] || revision=99
+ jq -cn --arg kind "$kind" --arg reason "$reason" --argjson revision "$revision" \
+ '{event:"networkpolicy-recovery-decision",kind:$kind,decision:"skip",reason:$reason,contextRevision:$revision,serverRevision:3}' >> "$HCC_LOG_BUFFER"
+ if [[ "$variant" = admission ]]; then
+ printf '%s\n' '{"event":"networkpolicy-request","cause":"mcp-recovery","admission":"before-capture"}' >> "$HCC_LOG_BUFFER"
+ elif [[ "$variant" = trailing ]]; then
+ printf '%s\n' '{"event":"networkpolicy-pass-start","causes":["context-recovery"],"trailing":true}' >> "$HCC_LOG_BUFFER"
+ elif [[ "$variant" = periodic ]]; then
+ printf '%s\n' '{"event":"networkpolicy-pass-start","causes":["periodic-resync"]}' >> "$HCC_LOG_BUFFER"
+ fi
+ }
+ die() { printf '%s\n' "$*" >&2; exit 1; }
+ hcc_pr_a_controlled_omissions
+ [[ "$cuts" = 6 ]]
+ ) > "$tmp/controlled-$variant/output" 2>&1 || status=$?
+ [[ -f "$tmp/controlled-$variant/controlled-recovery-observation.jsonl" ]]
+ if [[ "$variant" = success ]]; then
+ [[ "$status" = 0 ]] || { cat "$tmp/controlled-$variant/output"; exit 1; }
+ jq -se '[.[]|select(.kind=="McpServer")]|length==3' "$tmp/controlled-$variant/controlled-recovery-observation.jsonl" >/dev/null
+ jq -se '[.[]|select(.kind=="Context")]|length==3' "$tmp/controlled-$variant/controlled-recovery-observation.jsonl" >/dev/null
+ else
+ [[ "$status" != 0 ]] || { echo "FAIL: controlled omission accepted $variant" >&2; exit 1; }
+ grep -q 'controlled identical recoveries failed' "$tmp/controlled-$variant/output"
+ fi
+done
+printf 'PASS: controlled omission requires 3 fresh skips per kind and rejects replay, wrong state, admissions, trailing, mutation and periodic overlap\n'