Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

test(hcc): enforce held runtime process evidence

Alfredo Lopez90f8b70f5265
diff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.json
index 22477625c..073b3f9b4 100644
--- a/host-context-controller/package-lock.json
+++ b/host-context-controller/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.363",
+ "version": "0.1.364",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-host-context-controller",
- "version": "0.1.363",
+ "version": "0.1.364",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@clerum/codex-catalog-projection": "file:../packages/codex-catalog-projection",
diff --git a/host-context-controller/package.json b/host-context-controller/package.json
index 9a6cea2b6..6986055db 100644
--- a/host-context-controller/package.json
+++ b/host-context-controller/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.363",
+ "version": "0.1.364",
"description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API",
"main": "dist/main.js",
"engines": {
diff --git a/host-context-controller/src/k8sClient.ts b/host-context-controller/src/k8sClient.ts
index c96c3d04d..d01c55f66 100644
--- a/host-context-controller/src/k8sClient.ts
+++ b/host-context-controller/src/k8sClient.ts
@@ -1759,9 +1759,9 @@ export class McpServerWatcher implements McpServerProvider {
void this.requestHostFleetReconcile('CommunicationChannel recovery', lifecycleGeneration)
return true
} catch (error) {
- console.error(
- '[K8s] CommunicationChannel cache recovery failed; stateless lifecycle remains held active:',
- error
+ hccLogger.error(
+ '[K8s] CommunicationChannel cache recovery failed; preserving durable Host lifecycle state:',
+ { err: error }
)
return false
}
@@ -3957,8 +3957,8 @@ export class McpServerWatcher implements McpServerProvider {
if (err) {
console.error('[K8s] CommunicationChannel watch error:', err)
}
- console.log(
- '[K8s] CommunicationChannel watch ended; holding stateless lifecycle active until snapshot recovery'
+ hccLogger.info(
+ '[K8s] CommunicationChannel watch ended; preserving durable Host lifecycle state until snapshot recovery'
)
void this.requestHostFleetReconcile(
'CommunicationChannel watch interruption',
diff --git a/host-context-controller/test/hostReconciler.lifecycle.test.ts b/host-context-controller/test/hostReconciler.lifecycle.test.ts
index 80d3d0692..f6b2e51e7 100644
--- a/host-context-controller/test/hostReconciler.lifecycle.test.ts
+++ b/host-context-controller/test/hostReconciler.lifecycle.test.ts
@@ -1063,7 +1063,9 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {
customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))
const applied = reconciler.buildDeployment(host)
applied.status = { readyReplicas: 1 }
- persistHostDeployment(appsApi, host, applied)
+ const live = persistHostDeployment(appsApi, host, applied)
+ const templateBefore = structuredClone(live().spec!.template)
+ const replicasBefore = live().spec!.replicas
coreApi.readNamespacedPersistentVolumeClaim.mockRejectedValueOnce(new Error('PVC read failed'))
coreApi.readNamespacedService.mockRejectedValueOnce(new Error('Service read failed'))
@@ -1072,23 +1074,38 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {
.mockRejectedValueOnce(new Error('ingress policy failed'))
const gfsEgress = vi
.spyOn(reconciler as any, 'ensureMcpHostGfsEgressNetworkPolicy')
- .mockResolvedValue(undefined)
+ .mockResolvedValue('up_to_date')
const codexEgress = vi
.spyOn(reconciler as any, 'reconcileMcpHostCodexProxyEgressNetworkPolicy')
- .mockResolvedValue(undefined)
+ .mockResolvedValue('up_to_date')
await reconciler.reconcile(host)
- expect(coreApi.readNamespacedPersistentVolumeClaim).toHaveBeenCalledOnce()
- expect(coreApi.readNamespacedService).toHaveBeenCalledOnce()
+ expect(coreApi.readNamespacedPersistentVolumeClaim).toHaveBeenCalledWith({
+ namespace: host.namespace,
+ name: `${host.name}-workspace`,
+ })
+ expect(coreApi.readNamespacedService).toHaveBeenCalledWith({
+ namespace: host.namespace,
+ name: host.name,
+ })
expect(ingress).toHaveBeenCalledOnce()
+ expect(ingress).toHaveBeenCalledWith(host)
expect(gfsEgress).toHaveBeenCalledOnce()
expect(codexEgress).toHaveBeenCalledOnce()
+ expect(live().spec!.replicas).toBe(replicasBefore)
+ expect(live().spec!.template).toEqual(templateBefore)
expect(reconciler.getStatus(host.name)).toMatchObject({
deployed: true,
ready: false,
- message: expect.stringContaining('Host runtime boundary incomplete'),
+ message: expect.stringContaining('Host PVC: PVC read failed'),
})
+ expect(reconciler.getStatus(host.name)?.message).toContain('Host Service: Service read failed')
+ expect(reconciler.getStatus(host.name)?.message).toContain(
+ 'mcp-host ingress: ingress policy failed'
+ )
+ expect(reconciler.getStatus(host.name)?.message).not.toContain('GFS egress')
+ expect(reconciler.getStatus(host.name)?.message).not.toContain('Codex proxy egress')
})
it('does not report a Deployment applied when create conflict is followed by a missing read', async () => {
diff --git a/scripts/minikube/t2-common.sh b/scripts/minikube/t2-common.sh
index 999055e55..5b3bb32db 100755
--- a/scripts/minikube/t2-common.sh
+++ b/scripts/minikube/t2-common.sh
@@ -1100,24 +1100,8 @@ t2_process_check() {
for pid_file in "$T2_PROFILE_ROOT/$T2_PROFILE"/pids/*.pid; do
[ -f "$pid_file" ] || continue
recorded_pid="$(sed -n '1p' "$pid_file" 2>/dev/null || true)"
- # Auto-clean definitively dead profile-owned PID records instead of
- # blocking the entire T2 certification. The T0 contract tests and
- # pre-gate-sync legitimately create port-forwards whose processes
- # terminate when their parent make exits; leaving stale records
- # makes every subsequent T2 run fail with PORT_FORWARD_CONFLICT.
- # A PID that is not running is safe to remove only when the record
- # still belongs to this exact profile.
if ! pf_owner_validate_pid "$recorded_pid" ||
! t2_pid_file_matches_process "$pid_file" "$recorded_pid"; then
- if pf_owner_validate_pid "$recorded_pid" &&
- ! ps -p "$recorded_pid" -o pid= >/dev/null 2>&1; then
- # The recorded process is definitively dead; clean the stale
- # record so T2 can start fresh port-forwards.
- printf '[minikube-t2] auto-cleaned dead port-forward record: %s (pid=%s)\n' \
- "$pid_file" "$recorded_pid" >&2
- rm -f -- "$pid_file"
- continue
- fi
T2_NEXT_COMMAND='repair the exact profile-owned port-forward record before rerunning T2'
t2_fail PORT_FORWARD_CONFLICT "registered port-forward ownership is stale or invalid: $pid_file"
return 1
diff --git a/scripts/tests/test-hcc-watch-recovery-gate.sh b/scripts/tests/test-hcc-watch-recovery-gate.sh
index a97110722..c2326519a 100755
--- a/scripts/tests/test-hcc-watch-recovery-gate.sh
+++ b/scripts/tests/test-hcc-watch-recovery-gate.sh
@@ -409,7 +409,7 @@ fi
# shellcheck disable=SC2016
marker_bindings=(
"$WATCH_GATE" 'CommunicationChannel watch ended;'
- "$HCC_K8S_CLIENT" 'CommunicationChannel watch ended; holding stateless lifecycle active'
+ "$HCC_K8S_CLIENT" 'CommunicationChannel watch ended; preserving durable Host lifecycle state'
"$WATCH_GATE" 'Starting CommunicationChannel watch'
"$HCC_K8S_CLIENT" '[K8s] Starting CommunicationChannel watch'
@@ -418,7 +418,7 @@ marker_bindings=(
"$HCC_K8S_CLIENT" 'Recovered ${snapshot.channels.length} CommunicationChannel(s) into cache'
"$WATCH_GATE" 'cache recovery failed;'
- "$HCC_K8S_CLIENT" 'CommunicationChannel cache recovery failed; stateless lifecycle remains held active'
+ "$HCC_K8S_CLIENT" 'CommunicationChannel cache recovery failed; preserving durable Host lifecycle state'
"$WATCH_GATE" 'Listing all CommunicationChannels in namespace ${CHANNEL_NS}'
"$HCC_K8S_CLIENT" 'Listing all CommunicationChannels in namespace ${config.channelsNamespace}'
@@ -436,7 +436,7 @@ marker_bindings=(
"$HCC_K8S_CLIENT" 'CommunicationChannel watch event: ${type} for ${cc.name}'
"$LOG_HELPER" 'CommunicationChannel watch ended;'
- "$HCC_K8S_CLIENT" 'CommunicationChannel watch ended; holding stateless lifecycle active'
+ "$HCC_K8S_CLIENT" 'CommunicationChannel watch ended; preserving durable Host lifecycle state'
"$LOG_HELPER" 'Recovered [0-9]+ CommunicationChannel\(s\) into cache'
"$HCC_K8S_CLIENT" 'Recovered ${snapshot.channels.length} CommunicationChannel(s) into cache'
@@ -1385,7 +1385,7 @@ else
fi
printf '%s\n' \
- '[K8s] CommunicationChannel watch ended; holding stateless lifecycle active until snapshot recovery' \
+ '[K8s] CommunicationChannel watch ended; preserving durable Host lifecycle state until snapshot recovery' \
'[K8s] Listing all Hosts in namespace mcp-host' \
'[K8s] Recovered 2 CommunicationChannel(s) into cache (ccCacheSynced=true)' \
'[K8s] Reconciling 5 Host(s) for lifecycle after CommunicationChannel recovery' \
@@ -1397,7 +1397,7 @@ else
fi
printf '%s\n' \
- '[K8s] CommunicationChannel watch ended; holding stateless lifecycle active until snapshot recovery' \
+ '[K8s] CommunicationChannel watch ended; preserving durable Host lifecycle state until snapshot recovery' \
'[K8s] Listing all Hosts in namespace mcp-host' \
'[K8s] Recovered 2 CommunicationChannel(s) into cache (ccCacheSynced=true)' \
'[K8s] Reconciling 5 Host(s) after Host watch recovery convergence' \
@@ -1409,7 +1409,7 @@ else
fi
printf '%s\n' \
- '[K8s] CommunicationChannel watch ended; holding stateless lifecycle active until snapshot recovery' \
+ '[K8s] CommunicationChannel watch ended; preserving durable Host lifecycle state until snapshot recovery' \
'[K8s] Listing all Hosts in namespace mcp-host' \
'[K8s] Reconciling 5 Host(s) after Host watch recovery convergence' \
'[K8s] Recovered 2 CommunicationChannel(s) into cache (ccCacheSynced=true)' \
@@ -1422,7 +1422,7 @@ fi
printf '%s\n' \
'[K8s] Listing all Hosts in namespace mcp-host' \
- '[K8s] CommunicationChannel watch ended; holding stateless lifecycle active until snapshot recovery' \
+ '[K8s] CommunicationChannel watch ended; preserving durable Host lifecycle state until snapshot recovery' \
'[K8s] Recovered 2 CommunicationChannel(s) into cache (ccCacheSynced=true)' \
'[K8s] Reconciling 5 Host(s) for lifecycle after CommunicationChannel recovery' \
'[K8s] Completed Host reconciliation after CommunicationChannel recovery' >"$MOCK_LOG_FILE"
@@ -1433,7 +1433,7 @@ else
fi
printf '%s\n' \
- '[K8s] CommunicationChannel watch ended; holding stateless lifecycle active until snapshot recovery' \
+ '[K8s] CommunicationChannel watch ended; preserving durable Host lifecycle state until snapshot recovery' \
'[K8s] Listing all Hosts in namespace mcp-host' \
'[K8s] Recovered 2 CommunicationChannel(s) into cache (ccCacheSynced=true)' \
'[K8s] Reconciling 4 Host(s) for lifecycle after CommunicationChannel recovery' \
@@ -1445,12 +1445,12 @@ else
fi
printf '%s\n' \
- '[K8s] CommunicationChannel watch ended; holding stateless lifecycle active until snapshot recovery' \
+ '[K8s] CommunicationChannel watch ended; preserving durable Host lifecycle state until snapshot recovery' \
'[K8s] Listing all Hosts in namespace mcp-host' \
'[K8s] Recovered 1 CommunicationChannel(s) into cache (ccCacheSynced=true)' \
'[K8s] Reconciling 4 Host(s) for lifecycle after CommunicationChannel recovery' \
'[K8s] Completed Host reconciliation after CommunicationChannel recovery' \
- '[K8s] CommunicationChannel watch ended; holding stateless lifecycle active until snapshot recovery' \
+ '[K8s] CommunicationChannel watch ended; preserving durable Host lifecycle state until snapshot recovery' \
'[K8s] Listing all Hosts in namespace mcp-host' \
'[K8s] Recovered 2 CommunicationChannel(s) into cache (ccCacheSynced=true)' \
'[K8s] Reconciling 5 Host(s) for lifecycle after CommunicationChannel recovery' \
diff --git a/scripts/tests/test-minikube-t2-process-owner.sh b/scripts/tests/test-minikube-t2-process-owner.sh
index a309f2ac5..595c27b69 100755
--- a/scripts/tests/test-minikube-t2-process-owner.sh
+++ b/scripts/tests/test-minikube-t2-process-owner.sh
@@ -57,7 +57,8 @@ case "$mode" in
pid=) suffix=pid ;;
*) exit 1 ;;
esac
-[ ! -f "${T2_PS_DATA_DIR:?}/${pid}.${suffix}" ] || cat "${T2_PS_DATA_DIR}/${pid}.${suffix}"
+[ -f "${T2_PS_DATA_DIR:?}/${pid}.${suffix}" ] || exit 1
+cat "${T2_PS_DATA_DIR}/${pid}.${suffix}"
EOF_PS
chmod +x "$PS_BIN/ps"
@@ -115,13 +116,21 @@ run_process_check "$TEST_DIR/exact.out"
# A registered record must remain visible to the verdict even when the live
# process disappears from ps(1). This protects the fail-closed ownership scan
-# from treating a stale/dead pidfile as "nothing to check".
-rm -f "$PS_DATA/$PID.command"
+# from treating a stale/dead pidfile as "nothing to check" or deleting it.
+rm -f "$PS_DATA/$PID.command" "$PS_DATA/$PID.pid"
+if T2_PS_DATA_DIR="$PS_DATA" PATH="$PS_BIN:$PATH" ps -p "$PID" -o pid= >/dev/null 2>&1; then
+ printf 'FAIL: dead-PID fixture did not make ps fail\n' >&2
+ exit 1
+fi
if run_process_check "$TEST_DIR/stale-record.out"; then
printf 'FAIL: stale registered port-forward record was ignored\n' >&2
exit 1
fi
grep -Fq PORT_FORWARD_CONFLICT "$TEST_DIR/stale-record.out"
+[ -f "$RECORD" ] || {
+ printf 'FAIL: final process check removed the dead owner record\n' >&2
+ exit 1
+}
set_process_fixture kubectl "$exact_command"
printf '%s\n' 'different process start' >"$PS_DATA/$PID.start"