Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(hcc): close policy lifecycle review findings

Alfredo Lopezf4d0b41d5656
diff --git a/.github/workflows/ci-public.yml b/.github/workflows/ci-public.yml
index 293ca3c97..15f01e67d 100644
--- a/.github/workflows/ci-public.yml
+++ b/.github/workflows/ci-public.yml
@@ -450,6 +450,11 @@ jobs:
- name: Test HCC watch-churn readiness gate contract
run: bash scripts/tests/test-hcc-watch-churn-readiness-gate.sh
+ - name: HCC policy lifecycle and restoration contracts
+ run: |
+ bash scripts/tests/test-hcc-watch-churn-empty-fleet.sh
+ bash scripts/tests/test-hcc-lifecycle-cleanup.sh
+
- name: Test HCC rollout readiness gate contract
run: bash scripts/tests/test-hcc-rollout-readiness-gate.sh
diff --git a/Makefile b/Makefile
index 49fc7e38d..e071cec30 100644
--- a/Makefile
+++ b/Makefile
@@ -676,7 +676,7 @@ minikube-t2-np08-hcc-authorization: minikube-t2 ## Run canonical T2 including th
.PHONY: minikube-t2-hcc-networkpolicy-lifecycle
minikube-t2-hcc-networkpolicy-lifecycle:
@T2_HEALTHCHECK_COMMAND='bash scripts/e2e/e2e-hcc-networkpolicy-lifecycle.sh' \
- T2_HEALTHCHECK_TIMEOUT_SECONDS=900 $(MAKE) minikube-t2
+ T2_HEALTHCHECK_TIMEOUT_SECONDS=900 T2_HEALTHCHECK_KILL_GRACE_SECONDS=300 $(MAKE) minikube-t2
.PHONY: minikube-t2-runtime
minikube-t2-runtime: ## Exact-head T2 after T0 and T1 already passed on this HEAD and profile
diff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.json
index 331343bce..7b7905790 100644
--- a/host-context-controller/package-lock.json
+++ b/host-context-controller/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.311",
+ "version": "0.1.312",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-host-context-controller",
- "version": "0.1.311",
+ "version": "0.1.312",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@clerum/codex-catalog-projection": "file:../packages/codex-catalog-projection",
diff --git a/host-context-controller/package.json b/host-context-controller/package.json
index bc676b887..eff44cdda 100644
--- a/host-context-controller/package.json
+++ b/host-context-controller/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.311",
+ "version": "0.1.312",
"description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API",
"main": "dist/main.js",
"engines": {
diff --git a/host-context-controller/src/k8sClient.policyEligibility.integration.test.ts b/host-context-controller/src/k8sClient.policyEligibility.integration.test.ts
new file mode 100644
index 000000000..d558af3eb
--- /dev/null
+++ b/host-context-controller/src/k8sClient.policyEligibility.integration.test.ts
@@ -0,0 +1,315 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import type { V1DeleteOptions, V1NetworkPolicy, V1ObjectMeta } from '@kubernetes/client-node'
+import { McpServerWatcher } from './k8sClient'
+import type { ContextCRD, McpServerCRD, McpServerCrdStatus } from './types'
+
+type ApiServer = Pick<McpServerCRD, 'spec' | 'status'> & { metadata: V1ObjectMeta }
+type StoredPolicy = V1NetworkPolicy & { metadata: V1ObjectMeta }
+
+const fixture = vi.hoisted(() => ({
+ policies: new Map<string, StoredPolicy>(),
+ servers: new Map<string, ApiServer>(),
+ revision: 0,
+ watch: vi.fn().mockResolvedValue({ abort: vi.fn() }),
+ list: vi.fn(),
+}))
+
+vi.mock('./config', () => ({
+ config: {
+ devMode: false,
+ namespace: 'mcp-server',
+ hostNamespace: 'mcp-host',
+ rpcProxyNamespace: 'rpc-proxy',
+ channelsNamespace: 'channels',
+ controlPlaneNamespace: 'control-plane',
+ port: 8081,
+ hostK8sRequestTimeoutMs: 30000,
+ runtimeNamespaces: ['mcp-server', 'mcp-host', 'rpc-proxy'],
+ netPolOrphanDeleteCap: 10,
+ netPolOrphanDeleteCapPercent: 100,
+ externalEgressDnsResolveTimeoutMs: 5000,
+ externalEgressMaxEntries: 128,
+ externalEgressOverlapSec: 300,
+ netPolResyncIntervalSec: 0,
+ netPolDefaultsResyncIntervalSec: 0,
+ externalEgressResyncIntervalSec: 0,
+ hostResyncIntervalSec: 0,
+ },
+}))
+
+vi.mock('@kubernetes/client-node', async importOriginal => {
+ const actual = await importOriginal<typeof import('@kubernetes/client-node')>()
+ const absent = () => Object.assign(new Error('fixture object absent'), { code: 404 })
+ const read = (name: string) => {
+ const server = fixture.servers.get(name)
+ if (!server) throw absent()
+ return structuredClone(server)
+ }
+ const custom = {
+ listNamespacedCustomObject: fixture.list,
+ getNamespacedCustomObject: async ({ name }: { name: string }) => read(name),
+ getNamespacedCustomObjectStatus: async ({ name }: { name: string }) => read(name),
+ patchNamespacedCustomObjectStatus: async ({
+ name,
+ body,
+ }: {
+ name: string
+ body: Array<{ op: string; path: string; value: unknown }>
+ }) => {
+ const server = read(name)
+ for (const change of body) {
+ if (change.op === 'test') {
+ const metadataField = change.path.slice('/metadata/'.length)
+ const metadata = server.metadata as Record<string, unknown>
+ if (!change.path.startsWith('/metadata/') || metadata[metadataField] !== change.value)
+ throw Object.assign(new Error('status identity conflict'), { code: 409 })
+ continue
+ }
+ if (
+ change.op !== 'add' ||
+ !['/status', '/status/conditions', '/status/resolvedEgressIPs'].includes(change.path)
+ )
+ throw new Error(`Unsupported fixture status mutation: ${change.op} ${change.path}`)
+ if (change.path === '/status') server.status = change.value as McpServerCrdStatus
+ if (change.path === '/status/conditions')
+ (server.status ??= {}).conditions = change.value as McpServerCrdStatus['conditions']
+ if (change.path === '/status/resolvedEgressIPs')
+ (server.status ??= {}).resolvedEgressIPs =
+ change.value as McpServerCrdStatus['resolvedEgressIPs']
+ }
+ server.metadata.resourceVersion = String(++fixture.revision)
+ fixture.servers.set(name, server)
+ return server
+ },
+ }
+ const save = (namespace: string, body: StoredPolicy) => {
+ const key = `${namespace}/${body.metadata.name}`
+ const old = fixture.policies.get(key)
+ const policy = structuredClone({
+ ...body,
+ metadata: {
+ ...body.metadata,
+ namespace,
+ uid: old?.metadata.uid ?? `policy-${++fixture.revision}`,
+ resourceVersion: String(++fixture.revision),
+ },
+ })
+ fixture.policies.set(key, policy)
+ return structuredClone(policy)
+ }
+ const networking = {
+ listNamespacedNetworkPolicy: async ({
+ namespace,
+ labelSelector,
+ }: {
+ namespace: string
+ labelSelector?: string
+ }) => ({
+ items: [...fixture.policies.values()]
+ .filter(
+ policy =>
+ policy.metadata.namespace === namespace &&
+ (!labelSelector ||
+ labelSelector.split(',').every(clause => {
+ const [key, value] = clause.split('=')
+ return policy.metadata.labels?.[key] === value
+ }))
+ )
+ .map(policy => structuredClone(policy)),
+ }),
+ readNamespacedNetworkPolicy: async ({
+ namespace,
+ name,
+ }: {
+ namespace: string
+ name: string
+ }) => {
+ const policy = fixture.policies.get(`${namespace}/${name}`)
+ if (!policy) throw absent()
+ return structuredClone(policy)
+ },
+ createNamespacedNetworkPolicy: async ({
+ namespace,
+ body,
+ }: {
+ namespace: string
+ body: StoredPolicy
+ }) => {
+ if (fixture.policies.has(`${namespace}/${body.metadata.name}`))
+ throw Object.assign(new Error('already exists'), { code: 409 })
+ return save(namespace, body)
+ },
+ replaceNamespacedNetworkPolicy: async ({
+ namespace,
+ body,
+ }: {
+ namespace: string
+ body: StoredPolicy
+ }) => save(namespace, body),
+ deleteNamespacedNetworkPolicy: async ({
+ namespace,
+ name,
+ body,
+ }: {
+ namespace: string
+ name: string
+ body: V1DeleteOptions
+ }) => {
+ const key = `${namespace}/${name}`,
+ policy = fixture.policies.get(key)
+ if (!policy) throw absent()
+ expect(body.preconditions).toEqual({
+ uid: policy.metadata.uid,
+ resourceVersion: policy.metadata.resourceVersion,
+ })
+ fixture.policies.delete(key)
+ return {}
+ },
+ }
+ return {
+ ...actual,
+ KubeConfig: class {
+ loadFromDefault() {}
+ makeApiClient(kind: unknown) {
+ if (kind === actual.CustomObjectsApi) return custom
+ if (kind === actual.NetworkingV1Api) return networking
+ return {}
+ }
+ },
+ Watch: class {
+ watch = fixture.watch
+ },
+ }
+})
+
+// Secret validation/runtime objects are a precondition of this policy test.
+// Preserve the actual startup runEffect contract (and hence its egress gate),
+// while keeping unrelated workload API writes out of this fixture.
+vi.mock('./reconciler', () => ({
+ McpServerReconciler: class {
+ setInventoryAuthority() {}
+ setResolveCurrentServer() {}
+ reconcile() {
+ return Promise.resolve()
+ }
+ reconcileDelete() {
+ return Promise.resolve()
+ }
+ getStatus() {
+ return { deployed: true, ready: true }
+ }
+ async fullReconcile(
+ servers: McpServerCRD[],
+ options: { runEffect: (name: string, work: () => Promise<void>) => Promise<void> }
+ ) {
+ for (const server of servers) await options.runEffect(server.name, async () => {})
+ }
+ },
+}))
+
+type Harness = {
+ contexts: Map<string, ContextCRD>
+ contextCacheSynced: boolean
+ mcpServerCacheSynced: boolean
+ initialConvergenceRuns: Map<string, { promise: Promise<void> }>
+ mcpServerReconciliationQueues: Map<string, { queue: { tail: Promise<void> } }>
+ contextReconciliationQueues: Map<string, { queue: { tail: Promise<void> } }>
+ getMcpServerWatchCallback(): (type: string, object: ApiServer) => Promise<void>
+ recoverMcpServerInventoryAndWatch(): Promise<boolean>
+}
+
+async function drain(state: Harness): Promise<void> {
+ // Await work already scheduled by the input. Never call a reconcile method
+ // to "drain": that would conceal a missing WATCH/LIST scheduling edge.
+ for (let pass = 0; pass < 30; pass++) {
+ await Promise.resolve()
+ const pending = [...state.initialConvergenceRuns.values()].map(run => run.promise)
+ for (const queues of [state.mcpServerReconciliationQueues, state.contextReconciliationQueues]) {
+ for (const entry of queues.values()) pending.push(entry.queue.tail)
+ }
+ if (pending.length === 0) return
+ await Promise.all(pending)
+ }
+ throw new Error('policy queues did not drain')
+}
+
+describe('WATCH and recovered LIST policy effects', () => {
+ let watcher: McpServerWatcher
+ beforeEach(() => {
+ vi.useFakeTimers()
+ fixture.policies.clear()
+ fixture.servers.clear()
+ fixture.revision = 0
+ fixture.list.mockImplementation(async ({ plural }: { plural: string }) => ({
+ metadata: { resourceVersion: `list-${++fixture.revision}` },
+ items:
+ plural === 'mcpservers'
+ ? [...fixture.servers.values()].map(value => structuredClone(value))
+ : [],
+ }))
+ })
+ afterEach(async () => {
+ await watcher?.stop()
+ vi.useRealTimers()
+ })
+
+ it.each(['WATCH', 'LIST'] as const)(
+ '%s drains revocation and recovery of all four families',
+ async source => {
+ watcher = new McpServerWatcher()
+ const state = watcher as unknown as Harness
+ state.contextCacheSynced = true
+ state.mcpServerCacheSynced = true
+ state.contexts.set('ctx', {
+ name: 'ctx',
+ namespace: 'mcp-server',
+ uid: 'ctx-uid',
+ generation: 1,
+ spec: { contextId: 'ctx', mcpServers: ['server'] },
+ })
+ const server: ApiServer = {
+ metadata: {
+ name: 'server',
+ namespace: 'mcp-server',
+ uid: 'server-uid',
+ generation: 1,
+ resourceVersion: '1',
+ },
+ spec: {
+ image: 'fixture:v1',
+ contextRef: 'ctx',
+ transport: { type: 'streamableHttp', port: 3000 },
+ envSecret: { name: 'fixture-env', keys: [] },
+ egressBindings: [{ cidr: '1.2.3.4/32', port: 443 }],
+ },
+ status: { conditions: [{ type: 'SecretResolved', status: 'True', reason: 'SecretFound' }] },
+ }
+ fixture.servers.set('server', server)
+ await state.getMcpServerWatchCallback()('ADDED', structuredClone(server))
+ await drain(state)
+ expect(fixture.policies.size).toBe(4)
+ const keys = [...fixture.policies.keys()].sort()
+
+ for (const [status, reason, count] of [
+ ['False', 'SecretNotFound', 0],
+ ['True', 'SecretFound', 4],
+ ] as const) {
+ const updated = {
+ ...server,
+ status: { conditions: [{ type: 'SecretResolved', status, reason }] },
+ }
+ fixture.servers.set('server', updated)
+ if (source === 'WATCH')
+ await state.getMcpServerWatchCallback()('MODIFIED', structuredClone(updated))
+ else expect(await state.recoverMcpServerInventoryAndWatch()).toBe(true)
+ await drain(state)
+ await vi.advanceTimersByTimeAsync(5000)
+ await drain(state)
+ expect(fixture.policies.size).toBe(count)
+ }
+ expect([...fixture.policies.keys()].sort()).toEqual(keys)
+ if (source === 'LIST')
+ expect(fixture.list).toHaveBeenCalledWith(expect.objectContaining({ plural: 'mcpservers' }))
+ }
+ )
+})
diff --git a/host-context-controller/src/networkPolicyReconciler.test.ts b/host-context-controller/src/networkPolicyReconciler.test.ts
index 385abb2f2..4f8ba0372 100644
--- a/host-context-controller/src/networkPolicyReconciler.test.ts
+++ b/host-context-controller/src/networkPolicyReconciler.test.ts
@@ -6468,7 +6468,9 @@ describe('NetworkPolicyReconciler', () => {
})
return {}
})
- const runtime = new McpServerReconciler({} as k8s.KubeConfig, {
+ const runtimeConfig = new k8s.KubeConfig()
+ vi.spyOn(runtimeConfig, 'makeApiClient').mockReturnValue(mockApi)
+ const runtime = new McpServerReconciler(runtimeConfig, {
assumeInventoryAuthorityWhenUnconfigured: true,
appsApi: asAppsApi(apps),
coreApi: asCoreApi(core),
@@ -6652,6 +6654,76 @@ describe('NetworkPolicyReconciler', () => {
expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()
})
+ it.each([
+ { limit: 'absolute', cap: 1, percent: 100, lane: 'external-egress' },
+ { limit: 'percentage', cap: 10, percent: 30, lane: 'external-egress' },
+ { limit: 'absolute', cap: 1, percent: 100, lane: 'context-allow' },
+ { limit: 'percentage', cap: 10, percent: 30, lane: 'context-allow' },
+ ])(
+ 'excludes present ineligible owners from the $limit cap in $lane',
+ async ({ cap, percent, lane }) => {
+ const f = lifecycle()
+ await f.run()
+ expect(f.store.size).toBe(4)
+ config.netPolOrphanDeleteCap = cap
+ config.netPolOrphanDeleteCapPercent = percent
+ const source = [...f.store.values()].find(
+ policy =>
+ policy.metadata?.namespace === 'mcp-server' &&
+ policy.metadata.labels?.[POLICY_TYPE_LABEL] === lane
+ )!
+ const orphan = structuredClone(source)
+ orphan.metadata = {
+ ...orphan.metadata,
+ name:
+ lane === 'external-egress' ? 'ext-egress-gone-1-2-3-4-32-443' : 'ctx-gone-lifecycle',
+ uid: 'orphan-policy',
+ resourceVersion: '1',
+ labels: { ...orphan.metadata!.labels },
+ }
+ if (lane === 'external-egress') {
+ orphan.metadata.labels![MCPSERVER_LABEL] = 'gone'
+ orphan.spec!.podSelector = { matchLabels: { [MCPSERVER_LABEL]: 'gone' } }
+ } else orphan.metadata.labels!['clerum.io/context'] = 'gone'
+ const key = 'mcp-server/' + orphan.metadata.name
+ f.store.set(key, orphan)
+ f.cache.set(f.initial.name, { ...f.initial, spec: { ...f.initial.spec, enabled: false } })
+ await f.run()
+ expect(f.store.has(key)).toBe(false)
+ expect(f.store.size).toBe(0)
+ }
+ )
+
+ it('keeps the real orphan cap while revoking a present ineligible owner', async () => {
+ const f = lifecycle()
+ await f.run()
+ config.netPolOrphanDeleteCap = 1
+ config.netPolOrphanDeleteCapPercent = 100
+ const source = [...f.store.values()].find(
+ policy => policy.metadata?.labels?.[POLICY_TYPE_LABEL] === 'external-egress'
+ )!
+ for (const name of ['gone-one', 'gone-two']) {
+ const orphan = structuredClone(source)
+ orphan.metadata = {
+ ...orphan.metadata,
+ name: `ext-egress-${name}-1-2-3-4-32-443`,
+ uid: name,
+ resourceVersion: '1',
+ labels: { ...orphan.metadata!.labels, [MCPSERVER_LABEL]: name },
+ }
+ orphan.spec!.podSelector = { matchLabels: { [MCPSERVER_LABEL]: name } }
+ f.store.set('mcp-server/' + orphan.metadata.name, orphan)
+ }
+ f.cache.set(f.initial.name, { ...f.initial, spec: { ...f.initial.spec, enabled: false } })
+ await f.run()
+ expect(f.store.size).toBe(2)
+ expect(
+ [...f.store.values()].every(policy =>
+ policy.metadata?.labels?.[MCPSERVER_LABEL]?.startsWith('gone-')
+ )
+ ).toBe(true)
+ })
+
it('restores allows when envSecret is removed with its failure condition unchanged', async () => {
const f = lifecycle()
await f.run()
diff --git a/host-context-controller/src/networkPolicyReconciler.ts b/host-context-controller/src/networkPolicyReconciler.ts
index d35782e62..69ae979c9 100644
--- a/host-context-controller/src/networkPolicyReconciler.ts
+++ b/host-context-controller/src/networkPolicyReconciler.ts
@@ -1440,7 +1440,10 @@ export class NetworkPolicyReconciler {
}
const desiredContextIds = new Set(contexts.map(c => c.spec.contextId))
- const desiredServerNames = new Set(servers.filter(runtimeDesired).map(s => s.name))
+ // Orphan limits protect against missing inventory, not explicit policy
+ // revocation for a present owner. Eligibility is handled by the safety
+ // pass below and must not consume another owner's orphan budget.
+ const presentServerNames = new Set(servers.map(server => server.name))
const recordOrphanDelete = (lane: SafetyInventoryLane, deleted: boolean): void => {
if (!deleted) return
recordSafetyPassRevocation()
@@ -1556,7 +1559,7 @@ export class NetworkPolicyReconciler {
}
const isExternalLaneOrphan = (policy: k8s.V1NetworkPolicy): boolean => {
const serverName = policy.metadata?.labels?.[MCPSERVER_LABEL]
- return !serverName || !desiredServerNames.has(serverName)
+ return !serverName || !presentServerNames.has(serverName)
}
const countExternalLane =
options.serverInventoryComplete !== false && serverCleanupAuthoritative()
@@ -1683,7 +1686,7 @@ export class NetworkPolicyReconciler {
)
continue
}
- if (!desiredServerNames.has(serverName)) {
+ if (!presentServerNames.has(serverName)) {
const name = policy.metadata?.name || ''
await runServerEffect(serverName, async () => {
if (
diff --git a/host-context-controller/src/reconciler.test.ts b/host-context-controller/src/reconciler.test.ts
index dbe51da3e..7acd98b72 100644
--- a/host-context-controller/src/reconciler.test.ts
+++ b/host-context-controller/src/reconciler.test.ts
@@ -4,6 +4,7 @@ import {
asAppsApi,
asCoreApi,
asCustomApi,
+ asNetworkingApi,
createMockAppsApi,
createMockCoreApi,
createMockCustomApi,
@@ -492,7 +493,11 @@ describe('PR-B B1 — validateSecret result shape', () => {
beforeEach(() => {
vi.clearAllMocks()
- reconciler = new McpServerReconciler({} as k8s.KubeConfig, {
+ const kubeConfig = new k8s.KubeConfig()
+ // Keep the forbidden API reachable: reintroducing the old optional
+ // NetworkingApi client must not pass by observing an unrelated mock.
+ vi.spyOn(kubeConfig, 'makeApiClient').mockReturnValue(asNetworkingApi(networkingApi))
+ reconciler = new McpServerReconciler(kubeConfig, {
assumeInventoryAuthorityWhenUnconfigured: true,
appsApi: asAppsApi(appsApi),
coreApi: asCoreApi(coreApi),
diff --git a/scripts/e2e/_lib/hcc-cleanup-command.mjs b/scripts/e2e/_lib/hcc-cleanup-command.mjs
new file mode 100644
index 000000000..497248c39
--- /dev/null
+++ b/scripts/e2e/_lib/hcc-cleanup-command.mjs
@@ -0,0 +1,27 @@
+#!/usr/bin/env node
+// Cleanup requests stay in the enclosing gate's process group so they cannot
+// outlive its final SIGKILL and mutation lease.
+import { spawn } from 'node:child_process'
+
+const [rawSeconds, command, ...args] = process.argv.slice(2)
+if (!/^[1-9][0-9]*$/.test(rawSeconds ?? '') || Number(rawSeconds) > 270 || !command) {
+ process.stderr.write('cleanup command requires a deadline from 1 to 270 seconds and a command\n')
+ process.exit(2)
+}
+const child = spawn(command, args, { detached: false, stdio: 'inherit', env: process.env })
+let timedOut = false
+const timer = setTimeout(() => {
+ timedOut = true
+ // Kill only this directly owned request. Descendants remain in the outer
+ // deadline runner's process group for final reaping.
+ child.kill('SIGKILL')
+}, Number(rawSeconds) * 1000)
+child.once('error', error => {
+ clearTimeout(timer)
+ process.stderr.write(`cleanup command failed to start: ${error.code ?? 'unknown'}\n`)
+ process.exitCode = error.code === 'ENOENT' ? 127 : 126
+})
+child.once('exit', (code, signal) => {
+ clearTimeout(timer)
+ process.exitCode = timedOut ? 124 : (code ?? (signal ? 1 : 0))
+})
diff --git a/scripts/e2e/_lib/hcc-networkpolicy-lifecycle.sh b/scripts/e2e/_lib/hcc-networkpolicy-lifecycle.sh
index 6c105a641..106a06455 100644
--- a/scripts/e2e/_lib/hcc-networkpolicy-lifecycle.sh
+++ b/scripts/e2e/_lib/hcc-networkpolicy-lifecycle.sh
@@ -187,12 +187,13 @@ np604_cleanup() {
# Consume the deletion events while the owning controller is still running.
# Removing the Context only after stopping HCC would leave its allows to a
# namespace-wide orphan sweep, whose safety cap correctly refuses this fleet.
+ [ "${HCC_CLEANUP_NONBLOCKING:-false}" = true ] && return 0
wait_until 120 'NP604 owner-driven cleanup before HCC stop' np604_resources_absent
}
np604_resources_absent() {
local remaining
- remaining="$(np604_kctl get deployment,service,networkpolicy -A \
+ remaining="$(np604_kctl get pod,deployment,service,networkpolicy -A \
-l "clerum.io/mcpserver in (${NP604_SERVER},${NP604_CONTROL})" -o name)" || return 1
[ -z "$remaining" ]
}
diff --git a/scripts/e2e/_lib/hcc-watch-churn-fixture.sh b/scripts/e2e/_lib/hcc-watch-churn-fixture.sh
index c568de800..7d2f21ad8 100755
--- a/scripts/e2e/_lib/hcc-watch-churn-fixture.sh
+++ b/scripts/e2e/_lib/hcc-watch-churn-fixture.sh
@@ -245,9 +245,13 @@ delete_synthetic_fleet() {
# Non-destructive restore: undo the env overrides and hostAliases redirect.
restore_hcc_after_churn() {
+ local failed=0
kctl set env deployment/"$HCC_DEPLOY" -n "$HCC_NS" \
- KUBERNETES_SERVICE_HOST- KUBERNETES_SERVICE_PORT- CONTEXT_MAPPER_K8S_API_CIDRS- >/dev/null || return 1
+ KUBERNETES_SERVICE_HOST- KUBERNETES_SERVICE_PORT- CONTEXT_MAPPER_K8S_API_CIDRS- >/dev/null || failed=1
kctl patch deployment/"$HCC_DEPLOY" -n "$HCC_NS" --type=merge \
- -p '{"spec":{"template":{"spec":{"hostAliases":null}}}}' >/dev/null || return 1
- [ -z "$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o jsonpath='{.spec.template.spec.hostAliases}')" ] || return 1
+ -p '{"spec":{"template":{"spec":{"hostAliases":null}}}}' >/dev/null || failed=1
+ local aliases
+ aliases="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o jsonpath='{.spec.template.spec.hostAliases}')" || failed=1
+ [ -z "$aliases" ] || failed=1
+ return "$failed"
}
diff --git a/scripts/e2e/_lib/hcc-watch-lifecycle-cleanup.sh b/scripts/e2e/_lib/hcc-watch-lifecycle-cleanup.sh
new file mode 100644
index 000000000..dd92d96e4
--- /dev/null
+++ b/scripts/e2e/_lib/hcc-watch-lifecycle-cleanup.sh
@@ -0,0 +1,148 @@
+#!/usr/bin/env bash
+
+HCC_CLEANUP_COMMAND_RUNNER="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)/hcc-cleanup-command.mjs"
+
+# Each request consumes the remaining phase budget, including transport stalls.
+# Never start a nested detached deadline runner in the cleanup process group.
+hcc_cleanup_kctl() {
+ local remaining request_seconds command_seconds arg is_delete=false is_rollout=false
+ remaining=$((HCC_CLEANUP_PHASE_DEADLINE - SECONDS))
+ [ "$remaining" -gt 0 ] || return 124
+ request_seconds=$remaining
+ [ "$request_seconds" -le 30 ] || request_seconds=30
+ command_seconds=$request_seconds
+ local args=()
+ for arg in "$@"; do
+ case "$arg" in
+ --request-timeout=*|--timeout=*|--wait=*) ;;
+ *) args+=("$arg") ;;
+ esac
+ [ "$arg" != delete ] || is_delete=true
+ [ "$arg" != rollout ] || is_rollout=true
+ done
+ # Acceptance is followed by absence checks below; finalizers must not spend
+ # the restoration budget. This override applies only inside lifecycle cleanup.
+ if [ "$is_delete" = true ]; then args+=(--wait=false); fi
+ if [ "$is_rollout" = true ]; then
+ command_seconds=$remaining
+ request_seconds=$remaining
+ args+=(--timeout="${remaining}s")
+ fi
+ node "$HCC_CLEANUP_COMMAND_RUNNER" "$command_seconds" "$KUBECTL_BIN" \
+ --context "$E2E_KUBECONTEXT" "${args[@]}" \
+ --request-timeout="${request_seconds}s"
+}
+
+hcc_cleanup_wait_until() {
+ local requested=$1 description=$2 deadline remaining
+ shift 2
+ deadline=$((SECONDS + requested))
+ [ "$deadline" -le "$HCC_CLEANUP_PHASE_DEADLINE" ] || deadline=$HCC_CLEANUP_PHASE_DEADLINE
+ while [ "$SECONDS" -lt "$deadline" ]; do
+ "$@" && return 0
+ remaining=$((deadline - SECONDS))
+ [ "$remaining" -gt 0 ] || break
+ sleep 1
+ done
+ printf 'HCC cleanup deadline reached: %s\n' "$description" >&2
+ return 1
+}
+
+hcc_lifecycle_fixture_absent() {
+ local remaining host
+ if [ "$NP604_CREATED" = 1 ]; then
+ np604_resources_absent || return 1
+ remaining="$(kctl get mcpserver,secret -n "$MCP_NS" \
+ -l "e2e.clerum.io/suite=hcc-np604,e2e.clerum.io/run=${RUN_ID}" -o name)" || return 1
+ [ -z "$remaining" ] || return 1
+ fi
+ remaining="$(kctl get mcpserver,context,host -A \
+ -l "e2e.clerum.io/suite=hcc-watch-churn,e2e.clerum.io/run=${RUN_ID}" -o name)" || return 1
+ [ -z "$remaining" ] || return 1
+ while IFS= read -r host; do
+ [ -n "$host" ] || continue
+ remaining="$(kctl get pod,deployment,service,serviceaccount,role,rolebinding,secret,pvc,networkpolicy \
+ -A -l "clerum.io/host=${host}" -o name)" || return 1
+ [ -z "$remaining" ] || return 1
+ done <<<"$HCC_CLEANUP_HOSTS"
+ if [ "$FLEET_CREATED" = 1 ]; then
+ remaining="$(kctl get secret "$FLEET_SECRET" -n "$HOST_NS" --ignore-not-found -o name)" || return 1
+ [ -z "$remaining" ] || return 1
+ fi
+ if [ "$PROXY_CREATED" = 1 ]; then
+ remaining="$(kctl get deployment,service "$PROXY_NAME" -n "$HCC_NS" --ignore-not-found -o name)" || return 1
+ [ -z "$remaining" ] || return 1
+ remaining="$(kctl get pod -n "$HCC_NS" -l "app=${PROXY_NAME}" -o name)" || return 1
+ [ -z "$remaining" ] || return 1
+ fi
+ if [ "$PROBE_CREATED" = 1 ]; then
+ remaining="$(kctl get pod "$PROBE_NAME" -n "$HCC_NS" --ignore-not-found -o name)" || return 1
+ [ -z "$remaining" ] || return 1
+ fi
+ if [ "$PROXY_CREATED" = 1 ] || [ "$PROBE_CREATED" = 1 ]; then
+ remaining="$(kctl get networkpolicy "$PROXY_EGRESS_NP" "$HCC_PROXY_NP" "$PROBE_EGRESS_NP" \
+ -n "$HCC_NS" --ignore-not-found -o name)" || return 1
+ [ -z "$remaining" ] || return 1
+ fi
+ return 0
+}
+
+# Fixed budget: restore150 + fixtures100 + finalization20 = 270 seconds.
+# The lifecycle target reserves 300 seconds of runner grace. Restoring HCC
+# always precedes owner-driven fixture removal, which may never converge.
+cleanup_hcc_lifecycle() (
+ local status=$1 cleanup_failed=0 restore_ok=1 started=$SECONDS
+ local HCC_CLEANUP_PHASE_DEADLINE=$((SECONDS + 150)) HCC_CLEANUP_HOSTS=''
+ # Consumed through Bash dynamic scope by np604_cleanup in the sourced helper.
+ # shellcheck disable=SC2034
+ local HCC_CLEANUP_NONBLOCKING=true
+ trap - EXIT
+ trap ':' TERM INT HUP QUIT
+ set +e
+ kctl() { hcc_cleanup_kctl "$@"; }
+ wait_until() { hcc_cleanup_wait_until "$@"; }
+
+ # These are this shell's own capture children. Do not wait on a stalled stream.
+ [ -z "${HCC_LOG_STREAM_PID:-}" ] || kill "$HCC_LOG_STREAM_PID" 2>/dev/null
+ [ -z "${NP604_WATCH_PID:-}" ] || kill "$NP604_WATCH_PID" 2>/dev/null
+ NP604_WATCH_PID=''
+ if [ "$HCC_PATCHED" = 1 ]; then
+ restore_hcc_after_churn || restore_ok=0
+ fi
+ if [ "$HCC_SCALED_DOWN" = 1 ] || [ "$HCC_PATCHED" = 1 ]; then
+ kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" \
+ --replicas="${ORIGINAL_REPLICAS:-1}" >/dev/null 2>&1 || restore_ok=0
+ kctl rollout status deployment "$HCC_DEPLOY" -n "$HCC_NS" \
+ --timeout=150s >/dev/null 2>&1 || restore_ok=0
+ fi
+ if [ "$restore_ok" != 1 ]; then
+ print_repair_instructions
+ cleanup_failed=1
+ fi
+
+ # Preserve the proxy if HCC might still depend on its redirected API route.
+ if [ "$restore_ok" = 1 ]; then
+ HCC_CLEANUP_PHASE_DEADLINE=$((started + 250))
+ if [ "$FLEET_CREATED" = 1 ]; then
+ HCC_CLEANUP_HOSTS="$(kctl get host -A \
+ -l "e2e.clerum.io/suite=hcc-watch-churn,e2e.clerum.io/run=${RUN_ID}" \
+ -o 'jsonpath={range .items[*]}{.metadata.name}{"\n"}{end}')" || cleanup_failed=1
+ fi
+ np604_cleanup || cleanup_failed=1
+ [ "$FLEET_CREATED" != 1 ] || delete_synthetic_fleet || cleanup_failed=1
+ [ "$PROXY_CREATED" != 1 ] || kctl delete deployment,service "$PROXY_NAME" -n "$HCC_NS" --ignore-not-found >/dev/null 2>&1 || cleanup_failed=1
+ [ "$PROBE_CREATED" != 1 ] || kctl delete pod "$PROBE_NAME" -n "$HCC_NS" --ignore-not-found >/dev/null 2>&1 || cleanup_failed=1
+ if [ "$PROXY_CREATED" = 1 ] || [ "$PROBE_CREATED" = 1 ]; then
+ kctl delete networkpolicy "$PROXY_EGRESS_NP" "$HCC_PROXY_NP" "$PROBE_EGRESS_NP" \
+ -n "$HCC_NS" --ignore-not-found >/dev/null 2>&1 || cleanup_failed=1
+ fi
+ wait_until 100 'lifecycle fixture resources removed' hcc_lifecycle_fixture_absent || cleanup_failed=1
+ fi
+
+ HCC_CLEANUP_PHASE_DEADLINE=$((started + 270))
+ finalize_hcc_watch_gate_lock "$cleanup_failed" "$restore_ok" || cleanup_failed=1
+ print_results || cleanup_failed=1
+ if [ "$status" -eq 0 ] && [ "$cleanup_failed" != 0 ]; then status=1; fi
+ rm -f "$HCC_LOG_BUFFER" "$READY_SERIES"
+ return "$status"
+)
diff --git a/scripts/e2e/e2e-hcc-watch-churn-readiness.sh b/scripts/e2e/e2e-hcc-watch-churn-readiness.sh
index 8789640e4..4d314323b 100755
--- a/scripts/e2e/e2e-hcc-watch-churn-readiness.sh
+++ b/scripts/e2e/e2e-hcc-watch-churn-readiness.sh
@@ -45,6 +45,7 @@ case "${E2E_HCC_POLICY_LIFECYCLE:-0}" in
1)
# shellcheck source=scripts/e2e/_lib/hcc-networkpolicy-lifecycle.sh
source "${SCRIPT_DIR}/_lib/hcc-networkpolicy-lifecycle.sh"
+ source "${SCRIPT_DIR}/_lib/hcc-watch-lifecycle-cleanup.sh"
;;
*) echo "E2E_HCC_POLICY_LIFECYCLE must be 0 or 1" >&2; exit 2 ;;
esac
@@ -302,10 +303,11 @@ cleanup() {
local status=$? cleanup_failed=0 restore_ok=1
trap - EXIT
set +e
- stop_hcc_recovery_log_stream
if [ "${E2E_HCC_POLICY_LIFECYCLE:-0}" = 1 ]; then
- np604_cleanup || cleanup_failed=1
+ cleanup_hcc_lifecycle "$status"
+ exit $?
fi
+ stop_hcc_recovery_log_stream
if [ "$HCC_PATCHED" = 1 ]; then
kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" --replicas=0 >/dev/null 2>&1
wait_until 120 "HCC stop" hcc_pods_absent >/dev/null 2>&1 || restore_ok=0
@@ -327,9 +329,6 @@ cleanup() {
print_repair_instructions
cleanup_failed=1
}
- if [ "${E2E_HCC_POLICY_LIFECYCLE:-0}" = 1 ] && [ "$NP604_CREATED" = 1 ] && [ "$restore_ok" = 1 ]; then
- wait_until 120 "NP604 fixture resources removed by owners" np604_resources_absent || cleanup_failed=1
- fi
finalize_hcc_watch_gate_lock "$cleanup_failed" "$restore_ok" || cleanup_failed=1
print_results || status=1
[ "$cleanup_failed" = 0 ] || status=1
@@ -337,6 +336,11 @@ cleanup() {
exit "$status"
}
trap cleanup EXIT
+# An explicit signal exit enters the same restore path and preserves failure.
+trap 'exit 143' TERM
+trap 'exit 130' INT
+trap 'exit 129' HUP
+trap 'exit 131' QUIT
# ── FASE A: guards + snapshot ──
require_branch_owned_hcc_gate "$HCC_NS"
diff --git a/scripts/minikube/t2.sh b/scripts/minikube/t2.sh
index 33affc033..7ac29d490 100755
--- a/scripts/minikube/t2.sh
+++ b/scripts/minikube/t2.sh
@@ -22,6 +22,7 @@ T2_T0_COMMAND="$T2_T0_COMMAND"
T2_PLAYWRIGHT_COMMAND="$T2_PLAYWRIGHT_COMMAND"
T2_HEALTHCHECK_COMMAND="$T2_HEALTHCHECK_COMMAND"
T2_HEALTHCHECK_TIMEOUT_SECONDS="${T2_HEALTHCHECK_TIMEOUT_SECONDS:-120}"
+T2_HEALTHCHECK_KILL_GRACE_SECONDS="${T2_HEALTHCHECK_KILL_GRACE_SECONDS:-5}"
T2_DEADLINE_RUNNER="${T2_DEADLINE_RUNNER:-$SCRIPT_DIR/run-with-deadline.mjs}"
T2_PLAN_TMP=""
T2_T0_STATUS=NOT_RUN
@@ -352,6 +353,14 @@ run_np08_hcc_authorization() {
validate_healthcheck_contract() {
T2_HEALTHCHECK_REQUIRED=false
+ if ! [[ "$T2_HEALTHCHECK_KILL_GRACE_SECONDS" =~ ^[1-9][0-9]*$ ]] || \
+ [ "${#T2_HEALTHCHECK_KILL_GRACE_SECONDS}" -gt 3 ] || \
+ [ "$T2_HEALTHCHECK_KILL_GRACE_SECONDS" -gt 300 ]; then
+ T2_NEXT_COMMAND='set T2_HEALTHCHECK_KILL_GRACE_SECONDS to an integer from 1 to 300, then re-run T2'
+ t2_fail DEVELOPMENT_SCOPE_REQUIRED \
+ 'T2_HEALTHCHECK_KILL_GRACE_SECONDS must be an integer from 1 to 300'
+ return 1
+ fi
if ! [[ "$T2_HEALTHCHECK_TIMEOUT_SECONDS" =~ ^[1-9][0-9]*$ ]] || \
[ "$T2_HEALTHCHECK_TIMEOUT_SECONDS" -gt 900 ]; then
T2_NEXT_COMMAND='set T2_HEALTHCHECK_TIMEOUT_SECONDS to an integer from 1 to 900, then re-run T2'
@@ -388,7 +397,7 @@ run_healthcheck_if_requested() {
fi
node "$T2_DEADLINE_RUNNER" \
--timeout-seconds "$T2_HEALTHCHECK_TIMEOUT_SECONDS" \
- --heartbeat-seconds 20 --kill-grace-seconds 5 \
+ --heartbeat-seconds 20 --kill-grace-seconds "$T2_HEALTHCHECK_KILL_GRACE_SECONDS" \
--label t2-user-facing-health -- \
bash -c "$T2_HEALTHCHECK_COMMAND" || health_status=$?
if [ "$health_status" -eq 0 ]; then
diff --git a/scripts/tests/test-hcc-lifecycle-cleanup.sh b/scripts/tests/test-hcc-lifecycle-cleanup.sh
new file mode 100755
index 000000000..988f7bb65
--- /dev/null
+++ b/scripts/tests/test-hcc-lifecycle-cleanup.sh
@@ -0,0 +1,200 @@
+#!/usr/bin/env bash
+set -euo pipefail
+ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"
+tmp="$(mktemp -d)"
+trap 'rm -rf -- "$tmp"' EXIT
+# Evaluate the production declaration and validator, without sourcing T2's
+# orchestrator or contacting a cluster.
+source_function() {
+ local text
+ text="$(sed -n "/^$2() {/,/^}/p" "$1")"
+ [[ -n "$text" ]] || { echo "Missing function $2" >&2; exit 1; }
+ eval "$text"
+}
+unset T2_HEALTHCHECK_KILL_GRACE_SECONDS
+eval "$(sed -n '/^T2_HEALTHCHECK_KILL_GRACE_SECONDS=/p' "$ROOT/scripts/minikube/t2.sh")"
+[[ "$T2_HEALTHCHECK_KILL_GRACE_SECONDS" = 5 ]]
+source_function "$ROOT/scripts/minikube/t2.sh" validate_healthcheck_contract
+T2_HEALTHCHECK_TIMEOUT_SECONDS=900 T2_PLAN_STATE=already-synced
+T2_HEALTHCHECK_PENDING=false T2_HEALTHCHECK_COMMAND=''
+t2_fail() { printf '%s\n' "$*" >> "$tmp/validation-errors"; }
+for value in 0 -1 1.5 abc 301 999999999999999999999; do
+ T2_HEALTHCHECK_KILL_GRACE_SECONDS="$value"
+ if validate_healthcheck_contract; then
+ echo "FAIL: accepted invalid health cleanup grace $value" >&2; exit 1
+ fi
+done
+for value in 1 5 300; do
+ T2_HEALTHCHECK_KILL_GRACE_SECONDS="$value"
+ validate_healthcheck_contract
+done
+# Verify the validated value reaches the actual runner invocation.
+source_function "$ROOT/scripts/minikube/t2.sh" run_healthcheck_if_requested
+T2_HEALTHCHECK_COMMAND=true T2_DEADLINE_RUNNER="$ROOT/scripts/minikube/run-with-deadline.mjs"
+T2_HEALTHCHECK_KILL_GRACE_SECONDS=300
+node() { printf '%s\n' "$@" > "$tmp/runner-args"; }
+t2_evidence_write() { :; }
+run_healthcheck_if_requested
+unset -f node
+awk 'last=="--kill-grace-seconds" && $0=="300" {found=1} {last=$0} END {exit !found}' "$tmp/runner-args"
+echo 'PASS: health grace defaults to 5, validates 1..300 and reaches the runner'
+
+# Execute the actual callback and supervisor, replacing only the kubectl process.
+python3 - "$ROOT" "$tmp" <<'PY'
+import os
+from pathlib import Path
+import signal
+import subprocess
+import sys
+import time
+
+root, tmp = map(Path, sys.argv[1:])
+source = (root / 'scripts/e2e/e2e-hcc-watch-churn-readiness.sh').read_text()
+start = source.index('cleanup() {\n')
+end = source.index('\n}\n', start) + 3
+cleanup = source[start:end]
+traps = '\n'.join(line for line in source.splitlines() if line.startswith('trap '))
+assert 'trap cleanup EXIT' in traps and 'TERM' in traps
+kubectl = tmp / 'kubectl-fixture.py'
+kubectl.write_text('''#!/usr/bin/env python3
+import os, sys, time
+args = sys.argv[1:]
+with open(os.environ['TRACE'], 'a') as trace:
+ trace.write('kubectl ' + ' '.join(args) + '\\n')
+args = args[2:] # explicit --context fixture-context
+if args[:2] == ['set', 'env']:
+ time.sleep(float(os.environ.get('RESTORE_DELAY', '0')))
+if args[:2] == ['rollout', 'status'] and os.environ.get('FAIL_RESTORE') == '1':
+ sys.exit(1)
+if args[0] == 'delete' and os.environ.get('FAIL_FIXTURE') == '1':
+ sys.exit(1)
+''')
+kubectl.chmod(0o700)
+child = tmp / 'cleanup-child.sh'
+child.write_text('''#!/usr/bin/env bash
+set -euo pipefail
+ROOT=$1
+source "$ROOT/scripts/e2e/_lib/hcc-watch-churn-fixture.sh"
+source "$ROOT/scripts/e2e/_lib/hcc-networkpolicy-lifecycle.sh"
+source "$ROOT/scripts/e2e/_lib/hcc-watch-recovery-lock.sh"
+source "$ROOT/scripts/e2e/_lib/hcc-watch-lifecycle-cleanup.sh"
+E2E_HCC_POLICY_LIFECYCLE=1 E2E_KUBECONTEXT=fixture-context
+HCC_PATCHED=1 HCC_SCALED_DOWN=1 ORIGINAL_REPLICAS=1
+HCC_DEPLOY=fixture-hcc HCC_NS=fixture-control
+FLEET_CREATED=0 PROXY_CREATED=0 PROBE_CREATED=0
+NP604_CREATED=1 NP604_WATCH_PID=''
+NP604_SERVER=fixture-affected NP604_CONTROL=fixture-control MCP_NS=fixture-mcp
+RUN_ID=fixture HCC_GATE_LOCK_ACQUIRED=1
+HCC_LOG_BUFFER="${TRACE}.buffer" READY_SERIES="${TRACE}.series"
+release_hcc_watch_gate_lock() {
+ echo lock-released >> "$TRACE"
+ HCC_GATE_LOCK_ACQUIRED=0
+}
+print_hcc_watch_gate_lock_instructions() { echo lock-retained >> "$TRACE"; }
+print_repair_instructions() { echo repair-required >> "$TRACE"; }
+print_results() { :; }
+''' + cleanup + '\n' + traps + '''
+echo ready > "${TRACE}.ready"
+if [[ "${DIRECT_EXIT:-}" != '' ]]; then exit "$DIRECT_EXIT"; fi
+sleep 30
+''')
+runner = root / 'scripts/minikube/run-with-deadline.mjs'
+base_env = {**os.environ, 'KUBECTL_BIN': str(kubectl)}
+
+def run_case(name, *, mode='timeout', **extra):
+ trace = tmp / (name + '.trace')
+ env = {**base_env, 'TRACE': str(trace), **extra}
+ args = ['node', str(runner), '--timeout-seconds', '1' if mode == 'timeout' else '30',
+ '--kill-grace-seconds', '300', '--label', 'cleanup-contract', '--',
+ 'bash', str(child), str(root)]
+ process = subprocess.Popen(args, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+ if mode == 'term':
+ deadline = time.monotonic() + 5
+ ready = Path(str(trace) + '.ready')
+ while not ready.exists() and process.poll() is None and time.monotonic() < deadline:
+ time.sleep(0.02)
+ assert ready.exists(), 'child never became ready for TERM'
+ process.send_signal(signal.SIGTERM)
+ try:
+ stdout, stderr = process.communicate(timeout=25)
+ except subprocess.TimeoutExpired:
+ # Reap the owned process group even if this test fails.
+ process.send_signal(signal.SIGTERM)
+ try:
+ process.communicate(timeout=2)
+ except subprocess.TimeoutExpired:
+ process.send_signal(signal.SIGTERM)
+ process.communicate(timeout=3)
+ raise
+ expected = 124 if mode == 'timeout' else 143 if mode == 'term' else int(extra['DIRECT_EXIT'])
+ assert process.returncode == expected, (name, process.returncode, stdout, stderr)
+ lines = trace.read_text().splitlines()
+ restore = next(i for i, line in enumerate(lines) if ' rollout status ' in line)
+ fixture = [i for i, line in enumerate(lines) if ' delete mcpserver,secret ' in line]
+ if extra.get('FAIL_RESTORE') == '1':
+ assert not fixture, (name, 'removed fixtures before confirming restoration', lines)
+ else:
+ assert fixture and restore < fixture[0], (name, 'fixture removal preceded restoration', lines)
+ if extra.get('FAIL_RESTORE') == '1' or extra.get('FAIL_FIXTURE') == '1':
+ assert 'lock-retained' in lines and 'lock-released' not in lines, lines
+ else:
+ assert 'lock-released' in lines, (name, 'cleanup did not finish', lines, stderr)
+
+for mode in ('timeout', 'term'):
+ run_case(mode, mode=mode, RESTORE_DELAY='6')
+ print(f'PASS: {mode} retains its failure after real cleanup takes more than five seconds')
+run_case('restore-failure', mode='exit', DIRECT_EXIT='7', FAIL_RESTORE='1')
+run_case('fixture-failure', mode='exit', DIRECT_EXIT='7', FAIL_FIXTURE='1')
+print('PASS: restoration precedes fixture work; restoration/fixture failures retain the lock')
+
+# Deliberate transport stall, bounded by the production command wrapper.
+trace = tmp / 'budget.trace'
+env = {**base_env, 'TRACE': str(trace), 'RESTORE_DELAY': '5'}
+command = '''source "$1/scripts/e2e/_lib/hcc-watch-lifecycle-cleanup.sh"
+E2E_KUBECONTEXT=fixture-context
+HCC_CLEANUP_PHASE_DEADLINE=$((SECONDS + 1))
+hcc_cleanup_kctl set env deployment/fixture-hcc
+'''
+started = time.monotonic()
+result = subprocess.run(['bash', '-c', command, 'budget-check', str(root)], env=env,
+ capture_output=True, text=True, timeout=8)
+assert result.returncode == 124, (result.returncode, result.stderr)
+assert time.monotonic() - started < 4
+print('PASS: actual cleanup command boundary terminates a stalled request at its budget')
+PY
+
+# Actual absence predicates must handle partial setup and resources that lack
+# the suite/run labels (the proxy has an app label instead).
+(
+ set -euo pipefail
+ source "$ROOT/scripts/e2e/_lib/hcc-networkpolicy-lifecycle.sh"
+ source "$ROOT/scripts/e2e/_lib/hcc-watch-lifecycle-cleanup.sh"
+ NP604_CREATED=0 FLEET_CREATED=0 PROXY_CREATED=0 PROBE_CREATED=0
+ HCC_CLEANUP_HOSTS='' RUN_ID=fixture HCC_NS=fixture-control
+ unset NP604_SERVER NP604_CONTROL MCP_NS
+ kctl() { :; }
+ hcc_lifecycle_fixture_absent
+ echo 'PASS: absence check tolerates setup failure before NP604 names exist under nounset'
+
+ PROXY_CREATED=1 PROXY_NAME=fixture-proxy
+ PROXY_EGRESS_NP=fixture-egress HCC_PROXY_NP=fixture-hcc PROBE_EGRESS_NP=fixture-probe
+ remaining_kind=deployment
+ kctl() {
+ if [[ "$remaining_kind" = deployment && "$*" = "get deployment,service $PROXY_NAME -n $HCC_NS --ignore-not-found -o name" ]]; then
+ printf 'deployment.apps/%s\n' "$PROXY_NAME"
+ elif [[ "$remaining_kind" = pod && "$*" = "get pod -n $HCC_NS -l app=$PROXY_NAME -o name" ]]; then
+ printf 'pod/%s-terminating\n' "$PROXY_NAME"
+ fi
+ return 0
+ }
+ if hcc_lifecycle_fixture_absent; then
+ echo 'FAIL: exact-name proxy resource was mistaken for absence' >&2; exit 1
+ fi
+ remaining_kind=pod
+ if hcc_lifecycle_fixture_absent; then
+ echo 'FAIL: remaining app-labelled proxy pod was mistaken for absence' >&2; exit 1
+ fi
+ remaining_kind=none
+ hcc_lifecycle_fixture_absent
+ echo 'PASS: proxy absence requires both exact-name resources and app-labelled pods to disappear'
+)
diff --git a/scripts/tests/test-hcc-watch-churn-empty-fleet.sh b/scripts/tests/test-hcc-watch-churn-empty-fleet.sh
index cc215e336..43c8b5a6d 100755
--- a/scripts/tests/test-hcc-watch-churn-empty-fleet.sh
+++ b/scripts/tests/test-hcc-watch-churn-empty-fleet.sh
@@ -105,27 +105,5 @@ assert_hcc_sampled_recoveries
[[ ! -s "$tmp/sample-results" ]] # Disabled sampling must not produce vacuous PASS.
echo 'PASS: stress sampling remains required; focused mode makes no sampled-outage claim'
-(
- HCC_PATCHED=1 HCC_SCALED_DOWN=1 ORIGINAL_REPLICAS=1
- HCC_DEPLOY=host-context-controller HCC_NS=control-plane
- FLEET_CREATED=0 PROXY_CREATED=0 PROBE_CREATED=0
- E2E_HCC_POLICY_LIFECYCLE=1 NP604_CREATED=1 NP604_WATCH_PID=''
- NP604_SERVER=fixture-affected NP604_CONTROL=fixture-control
- PROXY_EGRESS_NP=fixture-egress HCC_PROXY_NP=fixture-hcc PROBE_EGRESS_NP=fixture-probe
- HCC_LOG_BUFFER="$tmp/buffer" READY_SERIES="$tmp/series"
- kctl() { printf '%s\n' "$*" >> "$tmp/order"; }
- stop_hcc_recovery_log_stream() { :; }
- hcc_pods_absent() { :; }
- wait_until() { shift 2; "$@"; }
- restore_hcc_after_churn() { :; }
- finalize_hcc_watch_gate_lock() { :; }
- print_results() { :; }
- eval "$cleanup_source"
- cleanup
-)
-awk '
- /get deployment,service,networkpolicy/ && !owner_check {owner_check=NR}
- /scale deployment host-context-controller.*--replicas=0/ && !stop {stop=NR}
- END {exit !(owner_check && stop && owner_check < stop)}
-' "$tmp/order"
-echo 'PASS: actual cleanup checks owner-driven fixture removal before stopping HCC'
+# Lifecycle timeout, restore-first ordering and retained-lock cases execute the
+# real cleanup and supervisor in test-hcc-lifecycle-cleanup.sh.