Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
test(hcc): enforce held runtime process evidence
Alfredo Lopez90f8b70f5265
diff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.jsonindex 22477625c..073b3f9b4 100644--- a/host-context-controller/package-lock.json+++ b/host-context-controller/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-host-context-controller",- "version": "0.1.363",+ "version": "0.1.364","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-host-context-controller",- "version": "0.1.363",+ "version": "0.1.364","license": "SEE LICENSE IN LICENSE","dependencies": {"@clerum/codex-catalog-projection": "file:../packages/codex-catalog-projection",diff --git a/host-context-controller/package.json b/host-context-controller/package.jsonindex 9a6cea2b6..6986055db 100644--- a/host-context-controller/package.json+++ b/host-context-controller/package.json@@ -1,6 +1,6 @@{"name": "clerum-host-context-controller",- "version": "0.1.363",+ "version": "0.1.364","description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API","main": "dist/main.js","engines": {diff --git a/host-context-controller/src/k8sClient.ts b/host-context-controller/src/k8sClient.tsindex c96c3d04d..d01c55f66 100644--- a/host-context-controller/src/k8sClient.ts+++ b/host-context-controller/src/k8sClient.ts@@ -1759,9 +1759,9 @@ export class McpServerWatcher implements McpServerProvider {void this.requestHostFleetReconcile('CommunicationChannel recovery', lifecycleGeneration)return true} catch (error) {- console.error(- '[K8s] CommunicationChannel cache recovery failed; stateless lifecycle remains held active:',- error+ hccLogger.error(+ '[K8s] CommunicationChannel cache recovery failed; preserving durable Host lifecycle state:',+ { err: error })return false}@@ -3957,8 +3957,8 @@ export class McpServerWatcher implements McpServerProvider {if (err) {console.error('[K8s] CommunicationChannel watch error:', err)}- console.log(- '[K8s] CommunicationChannel watch ended; holding stateless lifecycle active until snapshot recovery'+ hccLogger.info(+ '[K8s] CommunicationChannel watch ended; preserving durable Host lifecycle state until snapshot recovery')void this.requestHostFleetReconcile('CommunicationChannel watch interruption',diff --git a/host-context-controller/test/hostReconciler.lifecycle.test.ts b/host-context-controller/test/hostReconciler.lifecycle.test.tsindex 80d3d0692..f6b2e51e7 100644--- a/host-context-controller/test/hostReconciler.lifecycle.test.ts+++ b/host-context-controller/test/hostReconciler.lifecycle.test.ts@@ -1063,7 +1063,9 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))const applied = reconciler.buildDeployment(host)applied.status = { readyReplicas: 1 }- persistHostDeployment(appsApi, host, applied)+ const live = persistHostDeployment(appsApi, host, applied)+ const templateBefore = structuredClone(live().spec!.template)+ const replicasBefore = live().spec!.replicascoreApi.readNamespacedPersistentVolumeClaim.mockRejectedValueOnce(new Error('PVC read failed'))coreApi.readNamespacedService.mockRejectedValueOnce(new Error('Service read failed'))@@ -1072,23 +1074,38 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {.mockRejectedValueOnce(new Error('ingress policy failed'))const gfsEgress = vi.spyOn(reconciler as any, 'ensureMcpHostGfsEgressNetworkPolicy')- .mockResolvedValue(undefined)+ .mockResolvedValue('up_to_date')const codexEgress = vi.spyOn(reconciler as any, 'reconcileMcpHostCodexProxyEgressNetworkPolicy')- .mockResolvedValue(undefined)+ .mockResolvedValue('up_to_date')await reconciler.reconcile(host)- expect(coreApi.readNamespacedPersistentVolumeClaim).toHaveBeenCalledOnce()- expect(coreApi.readNamespacedService).toHaveBeenCalledOnce()+ expect(coreApi.readNamespacedPersistentVolumeClaim).toHaveBeenCalledWith({+ namespace: host.namespace,+ name: `${host.name}-workspace`,+ })+ expect(coreApi.readNamespacedService).toHaveBeenCalledWith({+ namespace: host.namespace,+ name: host.name,+ })expect(ingress).toHaveBeenCalledOnce()+ expect(ingress).toHaveBeenCalledWith(host)expect(gfsEgress).toHaveBeenCalledOnce()expect(codexEgress).toHaveBeenCalledOnce()+ expect(live().spec!.replicas).toBe(replicasBefore)+ expect(live().spec!.template).toEqual(templateBefore)expect(reconciler.getStatus(host.name)).toMatchObject({deployed: true,ready: false,- message: expect.stringContaining('Host runtime boundary incomplete'),+ message: expect.stringContaining('Host PVC: PVC read failed'),})+ expect(reconciler.getStatus(host.name)?.message).toContain('Host Service: Service read failed')+ expect(reconciler.getStatus(host.name)?.message).toContain(+ 'mcp-host ingress: ingress policy failed'+ )+ expect(reconciler.getStatus(host.name)?.message).not.toContain('GFS egress')+ expect(reconciler.getStatus(host.name)?.message).not.toContain('Codex proxy egress')})it('does not report a Deployment applied when create conflict is followed by a missing read', async () => {diff --git a/scripts/minikube/t2-common.sh b/scripts/minikube/t2-common.shindex 999055e55..5b3bb32db 100755--- a/scripts/minikube/t2-common.sh+++ b/scripts/minikube/t2-common.sh@@ -1100,24 +1100,8 @@ t2_process_check() {for pid_file in "$T2_PROFILE_ROOT/$T2_PROFILE"/pids/*.pid; do[ -f "$pid_file" ] || continuerecorded_pid="$(sed -n '1p' "$pid_file" 2>/dev/null || true)"- # Auto-clean definitively dead profile-owned PID records instead of- # blocking the entire T2 certification. The T0 contract tests and- # pre-gate-sync legitimately create port-forwards whose processes- # terminate when their parent make exits; leaving stale records- # makes every subsequent T2 run fail with PORT_FORWARD_CONFLICT.- # A PID that is not running is safe to remove only when the record- # still belongs to this exact profile.if ! pf_owner_validate_pid "$recorded_pid" ||! t2_pid_file_matches_process "$pid_file" "$recorded_pid"; then- if pf_owner_validate_pid "$recorded_pid" &&- ! ps -p "$recorded_pid" -o pid= >/dev/null 2>&1; then- # The recorded process is definitively dead; clean the stale- # record so T2 can start fresh port-forwards.- printf '[minikube-t2] auto-cleaned dead port-forward record: %s (pid=%s)\n' \- "$pid_file" "$recorded_pid" >&2- rm -f -- "$pid_file"- continue- fiT2_NEXT_COMMAND='repair the exact profile-owned port-forward record before rerunning T2't2_fail PORT_FORWARD_CONFLICT "registered port-forward ownership is stale or invalid: $pid_file"return 1diff --git a/scripts/tests/test-hcc-watch-recovery-gate.sh b/scripts/tests/test-hcc-watch-recovery-gate.shindex a97110722..c2326519a 100755--- a/scripts/tests/test-hcc-watch-recovery-gate.sh+++ b/scripts/tests/test-hcc-watch-recovery-gate.sh@@ -409,7 +409,7 @@ fi# shellcheck disable=SC2016marker_bindings=("$WATCH_GATE" 'CommunicationChannel watch ended;'- "$HCC_K8S_CLIENT" 'CommunicationChannel watch ended; holding stateless lifecycle active'+ "$HCC_K8S_CLIENT" 'CommunicationChannel watch ended; preserving durable Host lifecycle state'"$WATCH_GATE" 'Starting CommunicationChannel watch'"$HCC_K8S_CLIENT" '[K8s] Starting CommunicationChannel watch'@@ -418,7 +418,7 @@ marker_bindings=("$HCC_K8S_CLIENT" 'Recovered ${snapshot.channels.length} CommunicationChannel(s) into cache'"$WATCH_GATE" 'cache recovery failed;'- "$HCC_K8S_CLIENT" 'CommunicationChannel cache recovery failed; stateless lifecycle remains held active'+ "$HCC_K8S_CLIENT" 'CommunicationChannel cache recovery failed; preserving durable Host lifecycle state'"$WATCH_GATE" 'Listing all CommunicationChannels in namespace ${CHANNEL_NS}'"$HCC_K8S_CLIENT" 'Listing all CommunicationChannels in namespace ${config.channelsNamespace}'@@ -436,7 +436,7 @@ marker_bindings=("$HCC_K8S_CLIENT" 'CommunicationChannel watch event: ${type} for ${cc.name}'"$LOG_HELPER" 'CommunicationChannel watch ended;'- "$HCC_K8S_CLIENT" 'CommunicationChannel watch ended; holding stateless lifecycle active'+ "$HCC_K8S_CLIENT" 'CommunicationChannel watch ended; preserving durable Host lifecycle state'"$LOG_HELPER" 'Recovered [0-9]+ CommunicationChannel\(s\) into cache'"$HCC_K8S_CLIENT" 'Recovered ${snapshot.channels.length} CommunicationChannel(s) into cache'@@ -1385,7 +1385,7 @@ elsefiprintf '%s\n' \- '[K8s] CommunicationChannel watch ended; holding stateless lifecycle active until snapshot recovery' \+ '[K8s] CommunicationChannel watch ended; preserving durable Host lifecycle state until snapshot recovery' \'[K8s] Listing all Hosts in namespace mcp-host' \'[K8s] Recovered 2 CommunicationChannel(s) into cache (ccCacheSynced=true)' \'[K8s] Reconciling 5 Host(s) for lifecycle after CommunicationChannel recovery' \@@ -1397,7 +1397,7 @@ elsefiprintf '%s\n' \- '[K8s] CommunicationChannel watch ended; holding stateless lifecycle active until snapshot recovery' \+ '[K8s] CommunicationChannel watch ended; preserving durable Host lifecycle state until snapshot recovery' \'[K8s] Listing all Hosts in namespace mcp-host' \'[K8s] Recovered 2 CommunicationChannel(s) into cache (ccCacheSynced=true)' \'[K8s] Reconciling 5 Host(s) after Host watch recovery convergence' \@@ -1409,7 +1409,7 @@ elsefiprintf '%s\n' \- '[K8s] CommunicationChannel watch ended; holding stateless lifecycle active until snapshot recovery' \+ '[K8s] CommunicationChannel watch ended; preserving durable Host lifecycle state until snapshot recovery' \'[K8s] Listing all Hosts in namespace mcp-host' \'[K8s] Reconciling 5 Host(s) after Host watch recovery convergence' \'[K8s] Recovered 2 CommunicationChannel(s) into cache (ccCacheSynced=true)' \@@ -1422,7 +1422,7 @@ fiprintf '%s\n' \'[K8s] Listing all Hosts in namespace mcp-host' \- '[K8s] CommunicationChannel watch ended; holding stateless lifecycle active until snapshot recovery' \+ '[K8s] CommunicationChannel watch ended; preserving durable Host lifecycle state until snapshot recovery' \'[K8s] Recovered 2 CommunicationChannel(s) into cache (ccCacheSynced=true)' \'[K8s] Reconciling 5 Host(s) for lifecycle after CommunicationChannel recovery' \'[K8s] Completed Host reconciliation after CommunicationChannel recovery' >"$MOCK_LOG_FILE"@@ -1433,7 +1433,7 @@ elsefiprintf '%s\n' \- '[K8s] CommunicationChannel watch ended; holding stateless lifecycle active until snapshot recovery' \+ '[K8s] CommunicationChannel watch ended; preserving durable Host lifecycle state until snapshot recovery' \'[K8s] Listing all Hosts in namespace mcp-host' \'[K8s] Recovered 2 CommunicationChannel(s) into cache (ccCacheSynced=true)' \'[K8s] Reconciling 4 Host(s) for lifecycle after CommunicationChannel recovery' \@@ -1445,12 +1445,12 @@ elsefiprintf '%s\n' \- '[K8s] CommunicationChannel watch ended; holding stateless lifecycle active until snapshot recovery' \+ '[K8s] CommunicationChannel watch ended; preserving durable Host lifecycle state until snapshot recovery' \'[K8s] Listing all Hosts in namespace mcp-host' \'[K8s] Recovered 1 CommunicationChannel(s) into cache (ccCacheSynced=true)' \'[K8s] Reconciling 4 Host(s) for lifecycle after CommunicationChannel recovery' \'[K8s] Completed Host reconciliation after CommunicationChannel recovery' \- '[K8s] CommunicationChannel watch ended; holding stateless lifecycle active until snapshot recovery' \+ '[K8s] CommunicationChannel watch ended; preserving durable Host lifecycle state until snapshot recovery' \'[K8s] Listing all Hosts in namespace mcp-host' \'[K8s] Recovered 2 CommunicationChannel(s) into cache (ccCacheSynced=true)' \'[K8s] Reconciling 5 Host(s) for lifecycle after CommunicationChannel recovery' \diff --git a/scripts/tests/test-minikube-t2-process-owner.sh b/scripts/tests/test-minikube-t2-process-owner.shindex a309f2ac5..595c27b69 100755--- a/scripts/tests/test-minikube-t2-process-owner.sh+++ b/scripts/tests/test-minikube-t2-process-owner.sh@@ -57,7 +57,8 @@ case "$mode" inpid=) suffix=pid ;;*) exit 1 ;;esac-[ ! -f "${T2_PS_DATA_DIR:?}/${pid}.${suffix}" ] || cat "${T2_PS_DATA_DIR}/${pid}.${suffix}"+[ -f "${T2_PS_DATA_DIR:?}/${pid}.${suffix}" ] || exit 1+cat "${T2_PS_DATA_DIR}/${pid}.${suffix}"EOF_PSchmod +x "$PS_BIN/ps"@@ -115,13 +116,21 @@ run_process_check "$TEST_DIR/exact.out"# A registered record must remain visible to the verdict even when the live# process disappears from ps(1). This protects the fail-closed ownership scan-# from treating a stale/dead pidfile as "nothing to check".-rm -f "$PS_DATA/$PID.command"+# from treating a stale/dead pidfile as "nothing to check" or deleting it.+rm -f "$PS_DATA/$PID.command" "$PS_DATA/$PID.pid"+if T2_PS_DATA_DIR="$PS_DATA" PATH="$PS_BIN:$PATH" ps -p "$PID" -o pid= >/dev/null 2>&1; then+ printf 'FAIL: dead-PID fixture did not make ps fail\n' >&2+ exit 1+fiif run_process_check "$TEST_DIR/stale-record.out"; thenprintf 'FAIL: stale registered port-forward record was ignored\n' >&2exit 1figrep -Fq PORT_FORWARD_CONFLICT "$TEST_DIR/stale-record.out"+[ -f "$RECORD" ] || {+ printf 'FAIL: final process check removed the dead owner record\n' >&2+ exit 1+}set_process_fixture kubectl "$exact_command"printf '%s\n' 'different process start' >"$PS_DATA/$PID.start"