Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
test(hcc): verify omissions only after settled isolated recovery
Alfredo Lopeze1ee1e083d26
diff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.jsonindex 7b01d0815..049ea56b4 100644--- a/host-context-controller/package-lock.json+++ b/host-context-controller/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-host-context-controller",- "version": "0.1.315",+ "version": "0.1.316","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-host-context-controller",- "version": "0.1.315",+ "version": "0.1.316","license": "SEE LICENSE IN LICENSE","dependencies": {"@clerum/codex-catalog-projection": "file:../packages/codex-catalog-projection",diff --git a/host-context-controller/package.json b/host-context-controller/package.jsonindex ca8fc377d..ad1265e2a 100644--- a/host-context-controller/package.json+++ b/host-context-controller/package.json@@ -1,6 +1,6 @@{"name": "clerum-host-context-controller",- "version": "0.1.315",+ "version": "0.1.316","description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API","main": "dist/main.js","engines": {diff --git a/host-context-controller/src/k8sClient.policyEligibility.integration.test.ts b/host-context-controller/src/k8sClient.policyEligibility.integration.test.tsindex 947a53c4f..5c744aced 100644--- a/host-context-controller/src/k8sClient.policyEligibility.integration.test.ts+++ b/host-context-controller/src/k8sClient.policyEligibility.integration.test.ts@@ -880,4 +880,88 @@ describe('WATCH and recovered LIST policy effects', () => {await drain(state)expect(fixture.policies.get(desired[0])?.spec).toEqual(desired[1].spec)})+ it('A-T6 recovery scheduling: simultaneous retirement defers MCP until Context restores authority', async () => {+ const state = await settled()+ const before = structuredClone([...fixture.policies.entries()])+ expect(before).toHaveLength(17)+ const log = vi.spyOn(hccLogger, 'info')+ const originalList = fixture.list.getMockImplementation()!+ for (let cycle = 0; cycle < 3; cycle++) {+ await vi.advanceTimersByTimeAsync(7000)+ let release!: () => void, entered!: () => void+ const hold = new Promise<void>(resolve => {+ release = resolve+ })+ const started = new Promise<void>(resolve => {+ entered = resolve+ })+ fixture.list.mockImplementation(async args => {+ if (args.plural === 'contexts') {+ entered()+ await hold+ }+ return originalList(args)+ })+ ;(state as any).retireContextWatch()+ ;(state as any).retireMcpServerWatch()+ const contextRecovery = state.recoverContextInventoryAndWatch()+ await started+ expect(await state.recoverMcpServerInventoryAndWatch()).toBe(true)+ expect(state.contextCacheSynced).toBe(false)+ release()+ expect(await contextRecovery).toBe(true)+ await drain(state)+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(true)+ expect([...fixture.policies.entries()]).toEqual(before)+ }+ const decisions = log.mock.calls+ .map(([, data]) => data as any)+ .filter(data => data?.event === 'networkpolicy-recovery-decision')+ const mcps = decisions.filter(data => data.kind === 'McpServer')+ const contexts = decisions.filter(data => data.kind === 'Context')+ expect(mcps).toHaveLength(3)+ expect(mcps.every(data => data.decision === 'defer' && data.reason === 'no-authority')).toBe(+ true+ )+ expect(contexts).toHaveLength(3)+ expect(+ contexts.every(data => data.decision === 'skip' && data.reason === 'identical-complete')+ ).toBe(true)+ })++ it('A-T6 recovery scheduling: independent recoveries skip while their peer stays authoritative', async () => {+ const state = await settled()+ const before = structuredClone([...fixture.policies.entries()])+ expect(before).toHaveLength(17)+ const log = vi.spyOn(hccLogger, 'info')+ const admission = vi.spyOn(state, 'runInitialConvergence')+ for (let cycle = 0; cycle < 3; cycle++) {+ await vi.advanceTimersByTimeAsync(7000)+ expect(state.contextCacheSynced).toBe(true)+ ;(state as any).retireMcpServerWatch()+ expect(await state.recoverMcpServerInventoryAndWatch()).toBe(true)+ await drain(state)+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(true)+ expect([...fixture.policies.entries()]).toEqual(before)+ expect(state.mcpServerCacheSynced).toBe(true)+ ;(state as any).retireContextWatch()+ expect(await state.recoverContextInventoryAndWatch()).toBe(true)+ await drain(state)+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(true)+ expect([...fixture.policies.entries()]).toEqual(before)+ }+ const decisions = log.mock.calls+ .map(([, data]) => data as any)+ .filter(data => data?.event === 'networkpolicy-recovery-decision')+ const mcpSkips = decisions.filter(+ data => data.kind === 'McpServer' && data.decision === 'skip'+ ).length+ const contextSkips = decisions.filter(+ data => data.kind === 'Context' && data.decision === 'skip'+ ).length+ const npAdmissions = admission.mock.calls.filter(([lane]) => lane === 'NetworkPolicy').length+ expect(mcpSkips).toBe(3)+ expect(contextSkips).toBe(3)+ expect(npAdmissions).toBe(0)+ })})diff --git a/scripts/e2e/_lib/hcc-networkpolicy-lifecycle.sh b/scripts/e2e/_lib/hcc-networkpolicy-lifecycle.shindex c794dbd6f..0dee864d9 100755--- a/scripts/e2e/_lib/hcc-networkpolicy-lifecycle.sh+++ b/scripts/e2e/_lib/hcc-networkpolicy-lifecycle.sh@@ -145,8 +145,9 @@ np604_before_observation() {"e2e.clerum.io/observer=${RUN_ID}" --overwrite >/dev/nullwait_until 15 'NP604 policy observer positive witness' np604_observer_witness || die 'NP604 observer is not live'if [ "${E2E_HCC_PR_A:-0}" = 1 ]; then- HCC_PR_A_MCP_SKIP_BASE="$(hcc_pr_a_decision_count McpServer skip)"- HCC_PR_A_CONTEXT_SKIP_BASE="$(hcc_pr_a_decision_count Context skip)"+ # Consumed by the checkpoint helper in hcc-watch-pr-a.sh.+ # shellcheck disable=SC2034+ HCC_PR_A_OBSERVATION_LOG_LINE_BASE="$(wc -l < "$HCC_LOG_BUFFER" | tr -d ' ')"fiNP604_CUT_BASE="$(count_buffer '\[K8s\] Context watch ended; recovering authoritative inventory')"}@@ -154,9 +155,7 @@ np604_before_observation() {np604_after_observation() {np604_observer_alive || die 'NP604 observer stopped before completing observation'if [ "${E2E_HCC_PR_A:-0}" = 1 ]; then- [ "$(( $(hcc_pr_a_decision_count McpServer skip) - HCC_PR_A_MCP_SKIP_BASE ))" -ge 3 ] || die 'PR A requires 3 completed identical MCP omissions'- [ "$(( $(hcc_pr_a_decision_count Context skip) - HCC_PR_A_CONTEXT_SKIP_BASE ))" -ge 3 ] || die 'PR A requires 3 completed identical Context omissions'- printf 'PR_A_IDENTICAL_RECOVERIES=PASS (at least 3 per watch)\n'+ hcc_pr_a_recovery_checkpointfilocal current_cutscurrent_cuts="$(count_buffer '\[K8s\] Context watch ended; recovering authoritative inventory')"diff --git a/scripts/e2e/_lib/hcc-watch-lifecycle-cleanup.sh b/scripts/e2e/_lib/hcc-watch-lifecycle-cleanup.shindex 33383fdf2..f075eea36 100755--- a/scripts/e2e/_lib/hcc-watch-lifecycle-cleanup.sh+++ b/scripts/e2e/_lib/hcc-watch-lifecycle-cleanup.sh@@ -152,6 +152,14 @@ cleanup_hcc_lifecycle() (fiHCC_CLEANUP_PHASE_DEADLINE=$((started + 270))+ if { [ "$status" != 0 ] || [ "$cleanup_failed" != 0 ]; } &&+ [ "${E2E_HCC_PR_A:-0}" = 1 ] && [ "${NP604_CREATED:-0}" = 1 ] &&+ [ -d "${NP604_EVIDENCE:-}" ] && [ -r "$HCC_LOG_BUFFER" ]; then+ # Keep structured evidence for later phases too, before deleting the raw+ # buffer. Diagnostics do not replace or change the existing failure verdict.+ hcc_pr_a_recovery_checkpoint 0 "$NP604_EVIDENCE/failure-recovery-observation.jsonl" ||+ printf 'PR_A_FAILURE_CHECKPOINT=UNAVAILABLE\n' >&2+ fifinalize_hcc_watch_gate_lock "$cleanup_failed" "$restore_ok" || cleanup_failed=1print_results || cleanup_failed=1if [ "$status" -eq 0 ] && [ "$cleanup_failed" != 0 ]; then status=1; fidiff --git a/scripts/e2e/_lib/hcc-watch-pr-a.sh b/scripts/e2e/_lib/hcc-watch-pr-a.shindex 42d4ca858..7ba1585a5 100644--- a/scripts/e2e/_lib/hcc-watch-pr-a.sh+++ b/scripts/e2e/_lib/hcc-watch-pr-a.sh@@ -45,6 +45,80 @@ hcc_pr_a_logs() {jq -Rrc 'fromjson? | select(.event=="networkpolicy-recovery-decision" or .event=="networkpolicy-request" or .event=="networkpolicy-pass-start" or .event=="networkpolicy-pass-result")' "$HCC_LOG_BUFFER"}+hcc_pr_a_recovery_checkpoint() {+ # Defaults preserve the simultaneous-stress diagnostic checkpoint.+ local line_base=${1:-$HCC_PR_A_OBSERVATION_LOG_LINE_BASE}+ local destination=${2:-$NP604_EVIDENCE/recovery-observation.jsonl}+ # Persist only structured observation fields before a failing assertion can+ # enter cleanup and remove the original log buffer. Never persist payloads.+ tail -n "+$((line_base + 1))" "$HCC_LOG_BUFFER" |+ jq -Rrc 'fromjson? | select(.event=="networkpolicy-recovery-decision" or+ .event=="networkpolicy-request" or .event=="networkpolicy-pass-start" or .event=="networkpolicy-pass-result") |+ {ts,event,kind,decision,reason,contextRevision,serverRevision,passId,causes,trailing,result,cause,admission} |+ with_entries(select(.value!=null))' > "$destination"+ printf 'PR_A_RECOVERY_DECISION_COUNTS (observation window)\n'+ jq -sc '[.[]|select(.event=="networkpolicy-recovery-decision")] |+ group_by([.kind,.decision,.reason]) | map({kind:.[0].kind,decision:.[0].decision,reason:.[0].reason,count:length})' \+ "$destination"+}++hcc_pr_a_identical_skip_after() {+ local line_base=$1 kind=$2 revision=$3+ require_hcc_recovery_log_stream || return 1+ tail -n "+$((line_base + 1))" "$HCC_LOG_BUFFER" | jq -Rse --arg kind "$kind" --argjson revision "$revision" '+ split("\n") | map(fromjson?) | any(.event=="networkpolicy-recovery-decision" and .kind==$kind and+ .decision=="skip" and .reason=="identical-complete" and+ .contextRevision==$revision.contextRevision and .serverRevision==$revision.serverRevision)' >/dev/null+}++hcc_pr_a_no_recovery_admission_after() {+ tail -n "+$(($1 + 1))" "$HCC_LOG_BUFFER" | jq -Rse '+ split("\n") | map(fromjson?) | all(.[];+ ((.event=="networkpolicy-request" and (.cause=="mcp-recovery" or .cause=="context-recovery")) or+ (.event=="networkpolicy-pass-start" and any(.causes[]?; .=="mcp-recovery" or .=="context-recovery"))) | not)' >/dev/null+}++hcc_pr_a_controlled_omissions_core() {+ local line_base=$1 kind cycle before revision periodic affected control+ require_hcc_recovery_log_stream || return 1+ hcc_pr_a_gate 200 || return 1+ revision="$(hcc_pr_a_logs | jq -sce '[.[]|select(.event=="networkpolicy-pass-result" and .result=="certified")][-1] |+ {contextRevision,serverRevision} | select(.contextRevision!=null and .serverRevision!=null)')" || return 1+ periodic="$(hcc_pr_a_periodic_started_count)" || return 1+ affected="$(np604_snapshot "$NP604_SERVER")" || return 1+ control="$(np604_snapshot "$NP604_CONTROL")" || return 1+ [ "$(jq length <<< "$affected")" = 4 ] && [ "$(jq length <<< "$control")" = 4 ] || return 1+ for kind in McpServer Context; do+ for cycle in 1 2 3; do+ before="$(wc -l < "$HCC_LOG_BUFFER" | tr -d ' ')" || return 1+ hcc_pr_a_cut "$kind" || return 1+ wait_until 8 "fresh identical-complete ${kind} omission ${cycle}" \+ hcc_pr_a_identical_skip_after "$before" "$kind" "$revision" || return 1+ wait_until 5 'protected API recovered after controlled cut' hcc_pr_a_gate 200 || return 1+ hcc_pr_a_no_recovery_admission_after "$line_base" || return 1+ [ "$(hcc_pr_a_periodic_started_count)" = "$periodic" ] || {+ printf 'PR A controlled omission phase overlapped a periodic pass\n' >&2; return 1;+ }+ done+ done+ np604_invoke "$NP604_SERVER" || return 1+ np604_invoke "$NP604_CONTROL" || return 1+ [ "$(np604_snapshot "$NP604_SERVER")" = "$affected" ] || return 1+ [ "$(np604_snapshot "$NP604_CONTROL")" = "$control" ] || return 1+ [ "$(hcc_pr_a_periodic_started_count)" = "$periodic" ] || return 1+ require_hcc_recovery_log_stream || return 1+ hcc_pr_a_no_recovery_admission_after "$line_base"+}++hcc_pr_a_controlled_omissions() {+ local line_base status=0+ line_base="$(wc -l < "$HCC_LOG_BUFFER" | tr -d ' ')" || return 1+ hcc_pr_a_controlled_omissions_core "$line_base" || status=$?+ hcc_pr_a_recovery_checkpoint "$line_base" "$NP604_EVIDENCE/controlled-recovery-observation.jsonl"+ [ "$status" = 0 ] || die 'PR A controlled identical recoveries failed; checkpoint retained'+ printf 'PR_A_IDENTICAL_RECOVERIES=PASS (3 sequential cuts per watch; no recovery admission)\n'+}+hcc_pr_a_decision_after() {local baseline=$1 kind=$2 decision=$3require_hcc_recovery_log_stream || return 1@@ -87,6 +161,10 @@ hcc_pr_a_run() {periodic="$(hcc_pr_a_periodic_count)"wait_until 75 'PR A real periodic completion' hcc_pr_a_periodic_after "$periodic" || die 'PR A periodic path inactive'wait_until 15 'PR A periodic runtime effects complete' hcc_pr_a_runtime_completed || die 'PR A runtime did not complete before drift'+ # The B stress window may legitimately retain repair work. Test omission+ # separately after healthy binding, periodic policy/runtime completion and+ # the driver's acknowledged pause of automatic churn; never warm up on skips.+ hcc_pr_a_controlled_omissionsperiodic="$(hcc_pr_a_periodic_count)"context_spec="$(np604_kctl get mcpserver "$NP604_SERVER" -n "$MCP_NS" -o json | jq -Sc '{uid:.metadata.uid,generation:.metadata.generation,spec}')"hcc_pr_a_remove_servicediff --git a/scripts/tests/test-hcc-watch-pr-a.sh b/scripts/tests/test-hcc-watch-pr-a.shindex bfdb71335..31bcecf65 100644--- a/scripts/tests/test-hcc-watch-pr-a.sh+++ b/scripts/tests/test-hcc-watch-pr-a.sh@@ -18,6 +18,34 @@ if hcc_pr_a_decision_after 0 Context skip; then echo 'FAIL: deferral counted asif hcc_pr_a_decision_after 0 McpServer skip; then echo 'FAIL: empty observations passed'; exit 1; fiprintf 'PASS: recovery observation needs a new explicit omission of the exact kind\n'+# The unchanged real-cut stress assertion fails after a safe checkpoint.+# Deleting its raw buffer afterwards models the existing EXIT cleanup.+mkdir -p "$tmp/checkpoint"+printf '%s\n' 'before-observation' \+ '{"event":"networkpolicy-recovery-decision","kind":"McpServer","decision":"request","reason":"work-pending","contextRevision":2,"serverRevision":3,"body":"synthetic-private-payload"}' \+ '{"event":"networkpolicy-pass-start","passId":4,"causes":["mcp-recovery"],"contextRevision":2,"serverRevision":3,"headers":"synthetic-private-payload"}' > "$tmp/failed-runtime-buffer"+if (+ source "$ROOT/scripts/e2e/_lib/hcc-networkpolicy-lifecycle.sh"+ E2E_HCC_PR_A=1 NP604_CUT_BASE=0+ HCC_PR_A_OBSERVATION_LOG_LINE_BASE=1 HCC_LOG_BUFFER="$tmp/failed-runtime-buffer"+ NP604_EVIDENCE="$tmp/checkpoint"+ np604_observer_alive() { :; }+ count_buffer() { printf 0; }+ die() { printf '%s\n' "$*" >&2; exit 1; }+ np604_after_observation+) > "$tmp/checkpoint-result" 2>&1; then+ echo 'FAIL: zero real stress cuts unexpectedly passed' >&2; exit 1+fi+rm "$tmp/failed-runtime-buffer"+jq -se 'length==2 and .[0].reason=="work-pending" and .[1].passId==4 and+ all(.[]; has("body")|not) and all(.[]; has("headers")|not)' "$tmp/checkpoint/recovery-observation.jsonl" >/dev/null+grep -q 'NP604 needs three complete reconnect cycles' "$tmp/checkpoint-result"+grep -q '"reason":"work-pending","count":1' "$tmp/checkpoint-result"+if grep -q 'synthetic-private-payload' "$tmp/checkpoint-result" "$tmp/checkpoint/recovery-observation.jsonl"; then+ echo 'FAIL: payload leaked into checkpoint' >&2; exit 1+fi+printf 'PASS: unchanged stress assertion retains a projected checkpoint and grouped reasons after raw-buffer cleanup\n'+NP604_SERVER=fixture-affected MCP_NS=mcp-server HCC_PR_A_SERVICE_UID=oldnp604_kctl() { cat "$tmp/service"; }for variation in new old wrong-label wrong-port; do@@ -43,6 +71,7 @@ printf 'PASS: actual absence and new correctly scoped Service are required\n'(NP604_CONTROL=fixture-controlhcc_pr_a_gate() { :; }+ hcc_pr_a_controlled_omissions() { :; }np604_snapshot() { printf '[]'; }hcc_pr_a_periodic_count() { printf 1; }hcc_pr_a_periodic_started_count() { printf 1; }@@ -174,3 +203,57 @@ for mode in 0 1; dodonedoneprintf 'PASS: PR A retries within budget; legacy callers without wait_until remain strict and one-shot\n'++# Controlled omission has its own positive denominator: six real cut entries,+# then fresh same-kind/same-revision identical-complete decisions. Exercise+# the real phase body and checkpoint against hostile boundary outcomes.+for variant in success wrong-kind replay wrong-reason wrong-revision admission trailing mutation periodic; do+ mkdir -p "$tmp/controlled-$variant"+ printf '%s\n' '{"event":"networkpolicy-pass-result","result":"certified","contextRevision":2,"serverRevision":3,"causes":["startup"]}' > "$tmp/controlled-$variant/buffer"+ status=0+ (+ HCC_LOG_BUFFER="$tmp/controlled-$variant/buffer" NP604_EVIDENCE="$tmp/controlled-$variant"+ NP604_SERVER=fixture-affected NP604_CONTROL=fixture-control cuts=0+ require_hcc_recovery_log_stream() { :; }+ hcc_pr_a_gate() { :; }+ np604_invoke() { :; }+ np604_snapshot() {+ if [[ "$variant" = mutation && "$cuts" -gt 0 ]]; then+ printf '[{"name":"changed"},{},{},{}]'+ else+ printf '[{"name":"original"},{},{},{}]'+ fi+ }+ wait_until() { shift 2; "$@"; }+ hcc_pr_a_cut() {+ cuts=$((cuts + 1))+ local kind=$1 reason=identical-complete revision=2+ [[ "$variant" != replay ]] || return 0+ [[ "$variant" != wrong-kind ]] || kind=other+ [[ "$variant" != wrong-reason ]] || reason=work-pending+ [[ "$variant" != wrong-revision ]] || revision=99+ jq -cn --arg kind "$kind" --arg reason "$reason" --argjson revision "$revision" \+ '{event:"networkpolicy-recovery-decision",kind:$kind,decision:"skip",reason:$reason,contextRevision:$revision,serverRevision:3}' >> "$HCC_LOG_BUFFER"+ if [[ "$variant" = admission ]]; then+ printf '%s\n' '{"event":"networkpolicy-request","cause":"mcp-recovery","admission":"before-capture"}' >> "$HCC_LOG_BUFFER"+ elif [[ "$variant" = trailing ]]; then+ printf '%s\n' '{"event":"networkpolicy-pass-start","causes":["context-recovery"],"trailing":true}' >> "$HCC_LOG_BUFFER"+ elif [[ "$variant" = periodic ]]; then+ printf '%s\n' '{"event":"networkpolicy-pass-start","causes":["periodic-resync"]}' >> "$HCC_LOG_BUFFER"+ fi+ }+ die() { printf '%s\n' "$*" >&2; exit 1; }+ hcc_pr_a_controlled_omissions+ [[ "$cuts" = 6 ]]+ ) > "$tmp/controlled-$variant/output" 2>&1 || status=$?+ [[ -f "$tmp/controlled-$variant/controlled-recovery-observation.jsonl" ]]+ if [[ "$variant" = success ]]; then+ [[ "$status" = 0 ]] || { cat "$tmp/controlled-$variant/output"; exit 1; }+ jq -se '[.[]|select(.kind=="McpServer")]|length==3' "$tmp/controlled-$variant/controlled-recovery-observation.jsonl" >/dev/null+ jq -se '[.[]|select(.kind=="Context")]|length==3' "$tmp/controlled-$variant/controlled-recovery-observation.jsonl" >/dev/null+ else+ [[ "$status" != 0 ]] || { echo "FAIL: controlled omission accepted $variant" >&2; exit 1; }+ grep -q 'controlled identical recoveries failed' "$tmp/controlled-$variant/output"+ fi+done+printf 'PASS: controlled omission requires 3 fresh skips per kind and rejects replay, wrong state, admissions, trailing, mutation and periodic overlap\n'