Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(hcc): close policy lifecycle review findings
Alfredo Lopezf4d0b41d5656
diff --git a/.github/workflows/ci-public.yml b/.github/workflows/ci-public.ymlindex 293ca3c97..15f01e67d 100644--- a/.github/workflows/ci-public.yml+++ b/.github/workflows/ci-public.yml@@ -450,6 +450,11 @@ jobs:- name: Test HCC watch-churn readiness gate contractrun: bash scripts/tests/test-hcc-watch-churn-readiness-gate.sh+ - name: HCC policy lifecycle and restoration contracts+ run: |+ bash scripts/tests/test-hcc-watch-churn-empty-fleet.sh+ bash scripts/tests/test-hcc-lifecycle-cleanup.sh+- name: Test HCC rollout readiness gate contractrun: bash scripts/tests/test-hcc-rollout-readiness-gate.shdiff --git a/Makefile b/Makefileindex 49fc7e38d..e071cec30 100644--- a/Makefile+++ b/Makefile@@ -676,7 +676,7 @@ minikube-t2-np08-hcc-authorization: minikube-t2 ## Run canonical T2 including th.PHONY: minikube-t2-hcc-networkpolicy-lifecycleminikube-t2-hcc-networkpolicy-lifecycle:@T2_HEALTHCHECK_COMMAND='bash scripts/e2e/e2e-hcc-networkpolicy-lifecycle.sh' \- T2_HEALTHCHECK_TIMEOUT_SECONDS=900 $(MAKE) minikube-t2+ T2_HEALTHCHECK_TIMEOUT_SECONDS=900 T2_HEALTHCHECK_KILL_GRACE_SECONDS=300 $(MAKE) minikube-t2.PHONY: minikube-t2-runtimeminikube-t2-runtime: ## Exact-head T2 after T0 and T1 already passed on this HEAD and profilediff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.jsonindex 331343bce..7b7905790 100644--- a/host-context-controller/package-lock.json+++ b/host-context-controller/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-host-context-controller",- "version": "0.1.311",+ "version": "0.1.312","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-host-context-controller",- "version": "0.1.311",+ "version": "0.1.312","license": "SEE LICENSE IN LICENSE","dependencies": {"@clerum/codex-catalog-projection": "file:../packages/codex-catalog-projection",diff --git a/host-context-controller/package.json b/host-context-controller/package.jsonindex bc676b887..eff44cdda 100644--- a/host-context-controller/package.json+++ b/host-context-controller/package.json@@ -1,6 +1,6 @@{"name": "clerum-host-context-controller",- "version": "0.1.311",+ "version": "0.1.312","description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API","main": "dist/main.js","engines": {diff --git a/host-context-controller/src/k8sClient.policyEligibility.integration.test.ts b/host-context-controller/src/k8sClient.policyEligibility.integration.test.tsnew file mode 100644index 000000000..d558af3eb--- /dev/null+++ b/host-context-controller/src/k8sClient.policyEligibility.integration.test.ts@@ -0,0 +1,315 @@+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'+import type { V1DeleteOptions, V1NetworkPolicy, V1ObjectMeta } from '@kubernetes/client-node'+import { McpServerWatcher } from './k8sClient'+import type { ContextCRD, McpServerCRD, McpServerCrdStatus } from './types'++type ApiServer = Pick<McpServerCRD, 'spec' | 'status'> & { metadata: V1ObjectMeta }+type StoredPolicy = V1NetworkPolicy & { metadata: V1ObjectMeta }++const fixture = vi.hoisted(() => ({+ policies: new Map<string, StoredPolicy>(),+ servers: new Map<string, ApiServer>(),+ revision: 0,+ watch: vi.fn().mockResolvedValue({ abort: vi.fn() }),+ list: vi.fn(),+}))++vi.mock('./config', () => ({+ config: {+ devMode: false,+ namespace: 'mcp-server',+ hostNamespace: 'mcp-host',+ rpcProxyNamespace: 'rpc-proxy',+ channelsNamespace: 'channels',+ controlPlaneNamespace: 'control-plane',+ port: 8081,+ hostK8sRequestTimeoutMs: 30000,+ runtimeNamespaces: ['mcp-server', 'mcp-host', 'rpc-proxy'],+ netPolOrphanDeleteCap: 10,+ netPolOrphanDeleteCapPercent: 100,+ externalEgressDnsResolveTimeoutMs: 5000,+ externalEgressMaxEntries: 128,+ externalEgressOverlapSec: 300,+ netPolResyncIntervalSec: 0,+ netPolDefaultsResyncIntervalSec: 0,+ externalEgressResyncIntervalSec: 0,+ hostResyncIntervalSec: 0,+ },+}))++vi.mock('@kubernetes/client-node', async importOriginal => {+ const actual = await importOriginal<typeof import('@kubernetes/client-node')>()+ const absent = () => Object.assign(new Error('fixture object absent'), { code: 404 })+ const read = (name: string) => {+ const server = fixture.servers.get(name)+ if (!server) throw absent()+ return structuredClone(server)+ }+ const custom = {+ listNamespacedCustomObject: fixture.list,+ getNamespacedCustomObject: async ({ name }: { name: string }) => read(name),+ getNamespacedCustomObjectStatus: async ({ name }: { name: string }) => read(name),+ patchNamespacedCustomObjectStatus: async ({+ name,+ body,+ }: {+ name: string+ body: Array<{ op: string; path: string; value: unknown }>+ }) => {+ const server = read(name)+ for (const change of body) {+ if (change.op === 'test') {+ const metadataField = change.path.slice('/metadata/'.length)+ const metadata = server.metadata as Record<string, unknown>+ if (!change.path.startsWith('/metadata/') || metadata[metadataField] !== change.value)+ throw Object.assign(new Error('status identity conflict'), { code: 409 })+ continue+ }+ if (+ change.op !== 'add' ||+ !['/status', '/status/conditions', '/status/resolvedEgressIPs'].includes(change.path)+ )+ throw new Error(`Unsupported fixture status mutation: ${change.op} ${change.path}`)+ if (change.path === '/status') server.status = change.value as McpServerCrdStatus+ if (change.path === '/status/conditions')+ (server.status ??= {}).conditions = change.value as McpServerCrdStatus['conditions']+ if (change.path === '/status/resolvedEgressIPs')+ (server.status ??= {}).resolvedEgressIPs =+ change.value as McpServerCrdStatus['resolvedEgressIPs']+ }+ server.metadata.resourceVersion = String(++fixture.revision)+ fixture.servers.set(name, server)+ return server+ },+ }+ const save = (namespace: string, body: StoredPolicy) => {+ const key = `${namespace}/${body.metadata.name}`+ const old = fixture.policies.get(key)+ const policy = structuredClone({+ ...body,+ metadata: {+ ...body.metadata,+ namespace,+ uid: old?.metadata.uid ?? `policy-${++fixture.revision}`,+ resourceVersion: String(++fixture.revision),+ },+ })+ fixture.policies.set(key, policy)+ return structuredClone(policy)+ }+ const networking = {+ listNamespacedNetworkPolicy: async ({+ namespace,+ labelSelector,+ }: {+ namespace: string+ labelSelector?: string+ }) => ({+ items: [...fixture.policies.values()]+ .filter(+ policy =>+ policy.metadata.namespace === namespace &&+ (!labelSelector ||+ labelSelector.split(',').every(clause => {+ const [key, value] = clause.split('=')+ return policy.metadata.labels?.[key] === value+ }))+ )+ .map(policy => structuredClone(policy)),+ }),+ readNamespacedNetworkPolicy: async ({+ namespace,+ name,+ }: {+ namespace: string+ name: string+ }) => {+ const policy = fixture.policies.get(`${namespace}/${name}`)+ if (!policy) throw absent()+ return structuredClone(policy)+ },+ createNamespacedNetworkPolicy: async ({+ namespace,+ body,+ }: {+ namespace: string+ body: StoredPolicy+ }) => {+ if (fixture.policies.has(`${namespace}/${body.metadata.name}`))+ throw Object.assign(new Error('already exists'), { code: 409 })+ return save(namespace, body)+ },+ replaceNamespacedNetworkPolicy: async ({+ namespace,+ body,+ }: {+ namespace: string+ body: StoredPolicy+ }) => save(namespace, body),+ deleteNamespacedNetworkPolicy: async ({+ namespace,+ name,+ body,+ }: {+ namespace: string+ name: string+ body: V1DeleteOptions+ }) => {+ const key = `${namespace}/${name}`,+ policy = fixture.policies.get(key)+ if (!policy) throw absent()+ expect(body.preconditions).toEqual({+ uid: policy.metadata.uid,+ resourceVersion: policy.metadata.resourceVersion,+ })+ fixture.policies.delete(key)+ return {}+ },+ }+ return {+ ...actual,+ KubeConfig: class {+ loadFromDefault() {}+ makeApiClient(kind: unknown) {+ if (kind === actual.CustomObjectsApi) return custom+ if (kind === actual.NetworkingV1Api) return networking+ return {}+ }+ },+ Watch: class {+ watch = fixture.watch+ },+ }+})++// Secret validation/runtime objects are a precondition of this policy test.+// Preserve the actual startup runEffect contract (and hence its egress gate),+// while keeping unrelated workload API writes out of this fixture.+vi.mock('./reconciler', () => ({+ McpServerReconciler: class {+ setInventoryAuthority() {}+ setResolveCurrentServer() {}+ reconcile() {+ return Promise.resolve()+ }+ reconcileDelete() {+ return Promise.resolve()+ }+ getStatus() {+ return { deployed: true, ready: true }+ }+ async fullReconcile(+ servers: McpServerCRD[],+ options: { runEffect: (name: string, work: () => Promise<void>) => Promise<void> }+ ) {+ for (const server of servers) await options.runEffect(server.name, async () => {})+ }+ },+}))++type Harness = {+ contexts: Map<string, ContextCRD>+ contextCacheSynced: boolean+ mcpServerCacheSynced: boolean+ initialConvergenceRuns: Map<string, { promise: Promise<void> }>+ mcpServerReconciliationQueues: Map<string, { queue: { tail: Promise<void> } }>+ contextReconciliationQueues: Map<string, { queue: { tail: Promise<void> } }>+ getMcpServerWatchCallback(): (type: string, object: ApiServer) => Promise<void>+ recoverMcpServerInventoryAndWatch(): Promise<boolean>+}++async function drain(state: Harness): Promise<void> {+ // Await work already scheduled by the input. Never call a reconcile method+ // to "drain": that would conceal a missing WATCH/LIST scheduling edge.+ for (let pass = 0; pass < 30; pass++) {+ await Promise.resolve()+ const pending = [...state.initialConvergenceRuns.values()].map(run => run.promise)+ for (const queues of [state.mcpServerReconciliationQueues, state.contextReconciliationQueues]) {+ for (const entry of queues.values()) pending.push(entry.queue.tail)+ }+ if (pending.length === 0) return+ await Promise.all(pending)+ }+ throw new Error('policy queues did not drain')+}++describe('WATCH and recovered LIST policy effects', () => {+ let watcher: McpServerWatcher+ beforeEach(() => {+ vi.useFakeTimers()+ fixture.policies.clear()+ fixture.servers.clear()+ fixture.revision = 0+ fixture.list.mockImplementation(async ({ plural }: { plural: string }) => ({+ metadata: { resourceVersion: `list-${++fixture.revision}` },+ items:+ plural === 'mcpservers'+ ? [...fixture.servers.values()].map(value => structuredClone(value))+ : [],+ }))+ })+ afterEach(async () => {+ await watcher?.stop()+ vi.useRealTimers()+ })++ it.each(['WATCH', 'LIST'] as const)(+ '%s drains revocation and recovery of all four families',+ async source => {+ watcher = new McpServerWatcher()+ const state = watcher as unknown as Harness+ state.contextCacheSynced = true+ state.mcpServerCacheSynced = true+ state.contexts.set('ctx', {+ name: 'ctx',+ namespace: 'mcp-server',+ uid: 'ctx-uid',+ generation: 1,+ spec: { contextId: 'ctx', mcpServers: ['server'] },+ })+ const server: ApiServer = {+ metadata: {+ name: 'server',+ namespace: 'mcp-server',+ uid: 'server-uid',+ generation: 1,+ resourceVersion: '1',+ },+ spec: {+ image: 'fixture:v1',+ contextRef: 'ctx',+ transport: { type: 'streamableHttp', port: 3000 },+ envSecret: { name: 'fixture-env', keys: [] },+ egressBindings: [{ cidr: '1.2.3.4/32', port: 443 }],+ },+ status: { conditions: [{ type: 'SecretResolved', status: 'True', reason: 'SecretFound' }] },+ }+ fixture.servers.set('server', server)+ await state.getMcpServerWatchCallback()('ADDED', structuredClone(server))+ await drain(state)+ expect(fixture.policies.size).toBe(4)+ const keys = [...fixture.policies.keys()].sort()++ for (const [status, reason, count] of [+ ['False', 'SecretNotFound', 0],+ ['True', 'SecretFound', 4],+ ] as const) {+ const updated = {+ ...server,+ status: { conditions: [{ type: 'SecretResolved', status, reason }] },+ }+ fixture.servers.set('server', updated)+ if (source === 'WATCH')+ await state.getMcpServerWatchCallback()('MODIFIED', structuredClone(updated))+ else expect(await state.recoverMcpServerInventoryAndWatch()).toBe(true)+ await drain(state)+ await vi.advanceTimersByTimeAsync(5000)+ await drain(state)+ expect(fixture.policies.size).toBe(count)+ }+ expect([...fixture.policies.keys()].sort()).toEqual(keys)+ if (source === 'LIST')+ expect(fixture.list).toHaveBeenCalledWith(expect.objectContaining({ plural: 'mcpservers' }))+ }+ )+})diff --git a/host-context-controller/src/networkPolicyReconciler.test.ts b/host-context-controller/src/networkPolicyReconciler.test.tsindex 385abb2f2..4f8ba0372 100644--- a/host-context-controller/src/networkPolicyReconciler.test.ts+++ b/host-context-controller/src/networkPolicyReconciler.test.ts@@ -6468,7 +6468,9 @@ describe('NetworkPolicyReconciler', () => {})return {}})- const runtime = new McpServerReconciler({} as k8s.KubeConfig, {+ const runtimeConfig = new k8s.KubeConfig()+ vi.spyOn(runtimeConfig, 'makeApiClient').mockReturnValue(mockApi)+ const runtime = new McpServerReconciler(runtimeConfig, {assumeInventoryAuthorityWhenUnconfigured: true,appsApi: asAppsApi(apps),coreApi: asCoreApi(core),@@ -6652,6 +6654,76 @@ describe('NetworkPolicyReconciler', () => {expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()})+ it.each([+ { limit: 'absolute', cap: 1, percent: 100, lane: 'external-egress' },+ { limit: 'percentage', cap: 10, percent: 30, lane: 'external-egress' },+ { limit: 'absolute', cap: 1, percent: 100, lane: 'context-allow' },+ { limit: 'percentage', cap: 10, percent: 30, lane: 'context-allow' },+ ])(+ 'excludes present ineligible owners from the $limit cap in $lane',+ async ({ cap, percent, lane }) => {+ const f = lifecycle()+ await f.run()+ expect(f.store.size).toBe(4)+ config.netPolOrphanDeleteCap = cap+ config.netPolOrphanDeleteCapPercent = percent+ const source = [...f.store.values()].find(+ policy =>+ policy.metadata?.namespace === 'mcp-server' &&+ policy.metadata.labels?.[POLICY_TYPE_LABEL] === lane+ )!+ const orphan = structuredClone(source)+ orphan.metadata = {+ ...orphan.metadata,+ name:+ lane === 'external-egress' ? 'ext-egress-gone-1-2-3-4-32-443' : 'ctx-gone-lifecycle',+ uid: 'orphan-policy',+ resourceVersion: '1',+ labels: { ...orphan.metadata!.labels },+ }+ if (lane === 'external-egress') {+ orphan.metadata.labels![MCPSERVER_LABEL] = 'gone'+ orphan.spec!.podSelector = { matchLabels: { [MCPSERVER_LABEL]: 'gone' } }+ } else orphan.metadata.labels!['clerum.io/context'] = 'gone'+ const key = 'mcp-server/' + orphan.metadata.name+ f.store.set(key, orphan)+ f.cache.set(f.initial.name, { ...f.initial, spec: { ...f.initial.spec, enabled: false } })+ await f.run()+ expect(f.store.has(key)).toBe(false)+ expect(f.store.size).toBe(0)+ }+ )++ it('keeps the real orphan cap while revoking a present ineligible owner', async () => {+ const f = lifecycle()+ await f.run()+ config.netPolOrphanDeleteCap = 1+ config.netPolOrphanDeleteCapPercent = 100+ const source = [...f.store.values()].find(+ policy => policy.metadata?.labels?.[POLICY_TYPE_LABEL] === 'external-egress'+ )!+ for (const name of ['gone-one', 'gone-two']) {+ const orphan = structuredClone(source)+ orphan.metadata = {+ ...orphan.metadata,+ name: `ext-egress-${name}-1-2-3-4-32-443`,+ uid: name,+ resourceVersion: '1',+ labels: { ...orphan.metadata!.labels, [MCPSERVER_LABEL]: name },+ }+ orphan.spec!.podSelector = { matchLabels: { [MCPSERVER_LABEL]: name } }+ f.store.set('mcp-server/' + orphan.metadata.name, orphan)+ }+ f.cache.set(f.initial.name, { ...f.initial, spec: { ...f.initial.spec, enabled: false } })+ await f.run()+ expect(f.store.size).toBe(2)+ expect(+ [...f.store.values()].every(policy =>+ policy.metadata?.labels?.[MCPSERVER_LABEL]?.startsWith('gone-')+ )+ ).toBe(true)+ })+it('restores allows when envSecret is removed with its failure condition unchanged', async () => {const f = lifecycle()await f.run()diff --git a/host-context-controller/src/networkPolicyReconciler.ts b/host-context-controller/src/networkPolicyReconciler.tsindex d35782e62..69ae979c9 100644--- a/host-context-controller/src/networkPolicyReconciler.ts+++ b/host-context-controller/src/networkPolicyReconciler.ts@@ -1440,7 +1440,10 @@ export class NetworkPolicyReconciler {}const desiredContextIds = new Set(contexts.map(c => c.spec.contextId))- const desiredServerNames = new Set(servers.filter(runtimeDesired).map(s => s.name))+ // Orphan limits protect against missing inventory, not explicit policy+ // revocation for a present owner. Eligibility is handled by the safety+ // pass below and must not consume another owner's orphan budget.+ const presentServerNames = new Set(servers.map(server => server.name))const recordOrphanDelete = (lane: SafetyInventoryLane, deleted: boolean): void => {if (!deleted) returnrecordSafetyPassRevocation()@@ -1556,7 +1559,7 @@ export class NetworkPolicyReconciler {}const isExternalLaneOrphan = (policy: k8s.V1NetworkPolicy): boolean => {const serverName = policy.metadata?.labels?.[MCPSERVER_LABEL]- return !serverName || !desiredServerNames.has(serverName)+ return !serverName || !presentServerNames.has(serverName)}const countExternalLane =options.serverInventoryComplete !== false && serverCleanupAuthoritative()@@ -1683,7 +1686,7 @@ export class NetworkPolicyReconciler {)continue}- if (!desiredServerNames.has(serverName)) {+ if (!presentServerNames.has(serverName)) {const name = policy.metadata?.name || ''await runServerEffect(serverName, async () => {if (diff --git a/host-context-controller/src/reconciler.test.ts b/host-context-controller/src/reconciler.test.tsindex dbe51da3e..7acd98b72 100644--- a/host-context-controller/src/reconciler.test.ts+++ b/host-context-controller/src/reconciler.test.ts@@ -4,6 +4,7 @@ import {asAppsApi,asCoreApi,asCustomApi,+ asNetworkingApi,createMockAppsApi,createMockCoreApi,createMockCustomApi,@@ -492,7 +493,11 @@ describe('PR-B B1 — validateSecret result shape', () => {beforeEach(() => {vi.clearAllMocks()- reconciler = new McpServerReconciler({} as k8s.KubeConfig, {+ const kubeConfig = new k8s.KubeConfig()+ // Keep the forbidden API reachable: reintroducing the old optional+ // NetworkingApi client must not pass by observing an unrelated mock.+ vi.spyOn(kubeConfig, 'makeApiClient').mockReturnValue(asNetworkingApi(networkingApi))+ reconciler = new McpServerReconciler(kubeConfig, {assumeInventoryAuthorityWhenUnconfigured: true,appsApi: asAppsApi(appsApi),coreApi: asCoreApi(coreApi),diff --git a/scripts/e2e/_lib/hcc-cleanup-command.mjs b/scripts/e2e/_lib/hcc-cleanup-command.mjsnew file mode 100644index 000000000..497248c39--- /dev/null+++ b/scripts/e2e/_lib/hcc-cleanup-command.mjs@@ -0,0 +1,27 @@+#!/usr/bin/env node+// Cleanup requests stay in the enclosing gate's process group so they cannot+// outlive its final SIGKILL and mutation lease.+import { spawn } from 'node:child_process'++const [rawSeconds, command, ...args] = process.argv.slice(2)+if (!/^[1-9][0-9]*$/.test(rawSeconds ?? '') || Number(rawSeconds) > 270 || !command) {+ process.stderr.write('cleanup command requires a deadline from 1 to 270 seconds and a command\n')+ process.exit(2)+}+const child = spawn(command, args, { detached: false, stdio: 'inherit', env: process.env })+let timedOut = false+const timer = setTimeout(() => {+ timedOut = true+ // Kill only this directly owned request. Descendants remain in the outer+ // deadline runner's process group for final reaping.+ child.kill('SIGKILL')+}, Number(rawSeconds) * 1000)+child.once('error', error => {+ clearTimeout(timer)+ process.stderr.write(`cleanup command failed to start: ${error.code ?? 'unknown'}\n`)+ process.exitCode = error.code === 'ENOENT' ? 127 : 126+})+child.once('exit', (code, signal) => {+ clearTimeout(timer)+ process.exitCode = timedOut ? 124 : (code ?? (signal ? 1 : 0))+})diff --git a/scripts/e2e/_lib/hcc-networkpolicy-lifecycle.sh b/scripts/e2e/_lib/hcc-networkpolicy-lifecycle.shindex 6c105a641..106a06455 100644--- a/scripts/e2e/_lib/hcc-networkpolicy-lifecycle.sh+++ b/scripts/e2e/_lib/hcc-networkpolicy-lifecycle.sh@@ -187,12 +187,13 @@ np604_cleanup() {# Consume the deletion events while the owning controller is still running.# Removing the Context only after stopping HCC would leave its allows to a# namespace-wide orphan sweep, whose safety cap correctly refuses this fleet.+ [ "${HCC_CLEANUP_NONBLOCKING:-false}" = true ] && return 0wait_until 120 'NP604 owner-driven cleanup before HCC stop' np604_resources_absent}np604_resources_absent() {local remaining- remaining="$(np604_kctl get deployment,service,networkpolicy -A \+ remaining="$(np604_kctl get pod,deployment,service,networkpolicy -A \-l "clerum.io/mcpserver in (${NP604_SERVER},${NP604_CONTROL})" -o name)" || return 1[ -z "$remaining" ]}diff --git a/scripts/e2e/_lib/hcc-watch-churn-fixture.sh b/scripts/e2e/_lib/hcc-watch-churn-fixture.shindex c568de800..7d2f21ad8 100755--- a/scripts/e2e/_lib/hcc-watch-churn-fixture.sh+++ b/scripts/e2e/_lib/hcc-watch-churn-fixture.sh@@ -245,9 +245,13 @@ delete_synthetic_fleet() {# Non-destructive restore: undo the env overrides and hostAliases redirect.restore_hcc_after_churn() {+ local failed=0kctl set env deployment/"$HCC_DEPLOY" -n "$HCC_NS" \- KUBERNETES_SERVICE_HOST- KUBERNETES_SERVICE_PORT- CONTEXT_MAPPER_K8S_API_CIDRS- >/dev/null || return 1+ KUBERNETES_SERVICE_HOST- KUBERNETES_SERVICE_PORT- CONTEXT_MAPPER_K8S_API_CIDRS- >/dev/null || failed=1kctl patch deployment/"$HCC_DEPLOY" -n "$HCC_NS" --type=merge \- -p '{"spec":{"template":{"spec":{"hostAliases":null}}}}' >/dev/null || return 1- [ -z "$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o jsonpath='{.spec.template.spec.hostAliases}')" ] || return 1+ -p '{"spec":{"template":{"spec":{"hostAliases":null}}}}' >/dev/null || failed=1+ local aliases+ aliases="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o jsonpath='{.spec.template.spec.hostAliases}')" || failed=1+ [ -z "$aliases" ] || failed=1+ return "$failed"}diff --git a/scripts/e2e/_lib/hcc-watch-lifecycle-cleanup.sh b/scripts/e2e/_lib/hcc-watch-lifecycle-cleanup.shnew file mode 100644index 000000000..dd92d96e4--- /dev/null+++ b/scripts/e2e/_lib/hcc-watch-lifecycle-cleanup.sh@@ -0,0 +1,148 @@+#!/usr/bin/env bash++HCC_CLEANUP_COMMAND_RUNNER="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)/hcc-cleanup-command.mjs"++# Each request consumes the remaining phase budget, including transport stalls.+# Never start a nested detached deadline runner in the cleanup process group.+hcc_cleanup_kctl() {+ local remaining request_seconds command_seconds arg is_delete=false is_rollout=false+ remaining=$((HCC_CLEANUP_PHASE_DEADLINE - SECONDS))+ [ "$remaining" -gt 0 ] || return 124+ request_seconds=$remaining+ [ "$request_seconds" -le 30 ] || request_seconds=30+ command_seconds=$request_seconds+ local args=()+ for arg in "$@"; do+ case "$arg" in+ --request-timeout=*|--timeout=*|--wait=*) ;;+ *) args+=("$arg") ;;+ esac+ [ "$arg" != delete ] || is_delete=true+ [ "$arg" != rollout ] || is_rollout=true+ done+ # Acceptance is followed by absence checks below; finalizers must not spend+ # the restoration budget. This override applies only inside lifecycle cleanup.+ if [ "$is_delete" = true ]; then args+=(--wait=false); fi+ if [ "$is_rollout" = true ]; then+ command_seconds=$remaining+ request_seconds=$remaining+ args+=(--timeout="${remaining}s")+ fi+ node "$HCC_CLEANUP_COMMAND_RUNNER" "$command_seconds" "$KUBECTL_BIN" \+ --context "$E2E_KUBECONTEXT" "${args[@]}" \+ --request-timeout="${request_seconds}s"+}++hcc_cleanup_wait_until() {+ local requested=$1 description=$2 deadline remaining+ shift 2+ deadline=$((SECONDS + requested))+ [ "$deadline" -le "$HCC_CLEANUP_PHASE_DEADLINE" ] || deadline=$HCC_CLEANUP_PHASE_DEADLINE+ while [ "$SECONDS" -lt "$deadline" ]; do+ "$@" && return 0+ remaining=$((deadline - SECONDS))+ [ "$remaining" -gt 0 ] || break+ sleep 1+ done+ printf 'HCC cleanup deadline reached: %s\n' "$description" >&2+ return 1+}++hcc_lifecycle_fixture_absent() {+ local remaining host+ if [ "$NP604_CREATED" = 1 ]; then+ np604_resources_absent || return 1+ remaining="$(kctl get mcpserver,secret -n "$MCP_NS" \+ -l "e2e.clerum.io/suite=hcc-np604,e2e.clerum.io/run=${RUN_ID}" -o name)" || return 1+ [ -z "$remaining" ] || return 1+ fi+ remaining="$(kctl get mcpserver,context,host -A \+ -l "e2e.clerum.io/suite=hcc-watch-churn,e2e.clerum.io/run=${RUN_ID}" -o name)" || return 1+ [ -z "$remaining" ] || return 1+ while IFS= read -r host; do+ [ -n "$host" ] || continue+ remaining="$(kctl get pod,deployment,service,serviceaccount,role,rolebinding,secret,pvc,networkpolicy \+ -A -l "clerum.io/host=${host}" -o name)" || return 1+ [ -z "$remaining" ] || return 1+ done <<<"$HCC_CLEANUP_HOSTS"+ if [ "$FLEET_CREATED" = 1 ]; then+ remaining="$(kctl get secret "$FLEET_SECRET" -n "$HOST_NS" --ignore-not-found -o name)" || return 1+ [ -z "$remaining" ] || return 1+ fi+ if [ "$PROXY_CREATED" = 1 ]; then+ remaining="$(kctl get deployment,service "$PROXY_NAME" -n "$HCC_NS" --ignore-not-found -o name)" || return 1+ [ -z "$remaining" ] || return 1+ remaining="$(kctl get pod -n "$HCC_NS" -l "app=${PROXY_NAME}" -o name)" || return 1+ [ -z "$remaining" ] || return 1+ fi+ if [ "$PROBE_CREATED" = 1 ]; then+ remaining="$(kctl get pod "$PROBE_NAME" -n "$HCC_NS" --ignore-not-found -o name)" || return 1+ [ -z "$remaining" ] || return 1+ fi+ if [ "$PROXY_CREATED" = 1 ] || [ "$PROBE_CREATED" = 1 ]; then+ remaining="$(kctl get networkpolicy "$PROXY_EGRESS_NP" "$HCC_PROXY_NP" "$PROBE_EGRESS_NP" \+ -n "$HCC_NS" --ignore-not-found -o name)" || return 1+ [ -z "$remaining" ] || return 1+ fi+ return 0+}++# Fixed budget: restore150 + fixtures100 + finalization20 = 270 seconds.+# The lifecycle target reserves 300 seconds of runner grace. Restoring HCC+# always precedes owner-driven fixture removal, which may never converge.+cleanup_hcc_lifecycle() (+ local status=$1 cleanup_failed=0 restore_ok=1 started=$SECONDS+ local HCC_CLEANUP_PHASE_DEADLINE=$((SECONDS + 150)) HCC_CLEANUP_HOSTS=''+ # Consumed through Bash dynamic scope by np604_cleanup in the sourced helper.+ # shellcheck disable=SC2034+ local HCC_CLEANUP_NONBLOCKING=true+ trap - EXIT+ trap ':' TERM INT HUP QUIT+ set +e+ kctl() { hcc_cleanup_kctl "$@"; }+ wait_until() { hcc_cleanup_wait_until "$@"; }++ # These are this shell's own capture children. Do not wait on a stalled stream.+ [ -z "${HCC_LOG_STREAM_PID:-}" ] || kill "$HCC_LOG_STREAM_PID" 2>/dev/null+ [ -z "${NP604_WATCH_PID:-}" ] || kill "$NP604_WATCH_PID" 2>/dev/null+ NP604_WATCH_PID=''+ if [ "$HCC_PATCHED" = 1 ]; then+ restore_hcc_after_churn || restore_ok=0+ fi+ if [ "$HCC_SCALED_DOWN" = 1 ] || [ "$HCC_PATCHED" = 1 ]; then+ kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" \+ --replicas="${ORIGINAL_REPLICAS:-1}" >/dev/null 2>&1 || restore_ok=0+ kctl rollout status deployment "$HCC_DEPLOY" -n "$HCC_NS" \+ --timeout=150s >/dev/null 2>&1 || restore_ok=0+ fi+ if [ "$restore_ok" != 1 ]; then+ print_repair_instructions+ cleanup_failed=1+ fi++ # Preserve the proxy if HCC might still depend on its redirected API route.+ if [ "$restore_ok" = 1 ]; then+ HCC_CLEANUP_PHASE_DEADLINE=$((started + 250))+ if [ "$FLEET_CREATED" = 1 ]; then+ HCC_CLEANUP_HOSTS="$(kctl get host -A \+ -l "e2e.clerum.io/suite=hcc-watch-churn,e2e.clerum.io/run=${RUN_ID}" \+ -o 'jsonpath={range .items[*]}{.metadata.name}{"\n"}{end}')" || cleanup_failed=1+ fi+ np604_cleanup || cleanup_failed=1+ [ "$FLEET_CREATED" != 1 ] || delete_synthetic_fleet || cleanup_failed=1+ [ "$PROXY_CREATED" != 1 ] || kctl delete deployment,service "$PROXY_NAME" -n "$HCC_NS" --ignore-not-found >/dev/null 2>&1 || cleanup_failed=1+ [ "$PROBE_CREATED" != 1 ] || kctl delete pod "$PROBE_NAME" -n "$HCC_NS" --ignore-not-found >/dev/null 2>&1 || cleanup_failed=1+ if [ "$PROXY_CREATED" = 1 ] || [ "$PROBE_CREATED" = 1 ]; then+ kctl delete networkpolicy "$PROXY_EGRESS_NP" "$HCC_PROXY_NP" "$PROBE_EGRESS_NP" \+ -n "$HCC_NS" --ignore-not-found >/dev/null 2>&1 || cleanup_failed=1+ fi+ wait_until 100 'lifecycle fixture resources removed' hcc_lifecycle_fixture_absent || cleanup_failed=1+ fi++ HCC_CLEANUP_PHASE_DEADLINE=$((started + 270))+ finalize_hcc_watch_gate_lock "$cleanup_failed" "$restore_ok" || cleanup_failed=1+ print_results || cleanup_failed=1+ if [ "$status" -eq 0 ] && [ "$cleanup_failed" != 0 ]; then status=1; fi+ rm -f "$HCC_LOG_BUFFER" "$READY_SERIES"+ return "$status"+)diff --git a/scripts/e2e/e2e-hcc-watch-churn-readiness.sh b/scripts/e2e/e2e-hcc-watch-churn-readiness.shindex 8789640e4..4d314323b 100755--- a/scripts/e2e/e2e-hcc-watch-churn-readiness.sh+++ b/scripts/e2e/e2e-hcc-watch-churn-readiness.sh@@ -45,6 +45,7 @@ case "${E2E_HCC_POLICY_LIFECYCLE:-0}" in1)# shellcheck source=scripts/e2e/_lib/hcc-networkpolicy-lifecycle.shsource "${SCRIPT_DIR}/_lib/hcc-networkpolicy-lifecycle.sh"+ source "${SCRIPT_DIR}/_lib/hcc-watch-lifecycle-cleanup.sh";;*) echo "E2E_HCC_POLICY_LIFECYCLE must be 0 or 1" >&2; exit 2 ;;esac@@ -302,10 +303,11 @@ cleanup() {local status=$? cleanup_failed=0 restore_ok=1trap - EXITset +e- stop_hcc_recovery_log_streamif [ "${E2E_HCC_POLICY_LIFECYCLE:-0}" = 1 ]; then- np604_cleanup || cleanup_failed=1+ cleanup_hcc_lifecycle "$status"+ exit $?fi+ stop_hcc_recovery_log_streamif [ "$HCC_PATCHED" = 1 ]; thenkctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" --replicas=0 >/dev/null 2>&1wait_until 120 "HCC stop" hcc_pods_absent >/dev/null 2>&1 || restore_ok=0@@ -327,9 +329,6 @@ cleanup() {print_repair_instructionscleanup_failed=1}- if [ "${E2E_HCC_POLICY_LIFECYCLE:-0}" = 1 ] && [ "$NP604_CREATED" = 1 ] && [ "$restore_ok" = 1 ]; then- wait_until 120 "NP604 fixture resources removed by owners" np604_resources_absent || cleanup_failed=1- fifinalize_hcc_watch_gate_lock "$cleanup_failed" "$restore_ok" || cleanup_failed=1print_results || status=1[ "$cleanup_failed" = 0 ] || status=1@@ -337,6 +336,11 @@ cleanup() {exit "$status"}trap cleanup EXIT+# An explicit signal exit enters the same restore path and preserves failure.+trap 'exit 143' TERM+trap 'exit 130' INT+trap 'exit 129' HUP+trap 'exit 131' QUIT# ── FASE A: guards + snapshot ──require_branch_owned_hcc_gate "$HCC_NS"diff --git a/scripts/minikube/t2.sh b/scripts/minikube/t2.shindex 33affc033..7ac29d490 100755--- a/scripts/minikube/t2.sh+++ b/scripts/minikube/t2.sh@@ -22,6 +22,7 @@ T2_T0_COMMAND="$T2_T0_COMMAND"T2_PLAYWRIGHT_COMMAND="$T2_PLAYWRIGHT_COMMAND"T2_HEALTHCHECK_COMMAND="$T2_HEALTHCHECK_COMMAND"T2_HEALTHCHECK_TIMEOUT_SECONDS="${T2_HEALTHCHECK_TIMEOUT_SECONDS:-120}"+T2_HEALTHCHECK_KILL_GRACE_SECONDS="${T2_HEALTHCHECK_KILL_GRACE_SECONDS:-5}"T2_DEADLINE_RUNNER="${T2_DEADLINE_RUNNER:-$SCRIPT_DIR/run-with-deadline.mjs}"T2_PLAN_TMP=""T2_T0_STATUS=NOT_RUN@@ -352,6 +353,14 @@ run_np08_hcc_authorization() {validate_healthcheck_contract() {T2_HEALTHCHECK_REQUIRED=false+ if ! [[ "$T2_HEALTHCHECK_KILL_GRACE_SECONDS" =~ ^[1-9][0-9]*$ ]] || \+ [ "${#T2_HEALTHCHECK_KILL_GRACE_SECONDS}" -gt 3 ] || \+ [ "$T2_HEALTHCHECK_KILL_GRACE_SECONDS" -gt 300 ]; then+ T2_NEXT_COMMAND='set T2_HEALTHCHECK_KILL_GRACE_SECONDS to an integer from 1 to 300, then re-run T2'+ t2_fail DEVELOPMENT_SCOPE_REQUIRED \+ 'T2_HEALTHCHECK_KILL_GRACE_SECONDS must be an integer from 1 to 300'+ return 1+ fiif ! [[ "$T2_HEALTHCHECK_TIMEOUT_SECONDS" =~ ^[1-9][0-9]*$ ]] || \[ "$T2_HEALTHCHECK_TIMEOUT_SECONDS" -gt 900 ]; thenT2_NEXT_COMMAND='set T2_HEALTHCHECK_TIMEOUT_SECONDS to an integer from 1 to 900, then re-run T2'@@ -388,7 +397,7 @@ run_healthcheck_if_requested() {finode "$T2_DEADLINE_RUNNER" \--timeout-seconds "$T2_HEALTHCHECK_TIMEOUT_SECONDS" \- --heartbeat-seconds 20 --kill-grace-seconds 5 \+ --heartbeat-seconds 20 --kill-grace-seconds "$T2_HEALTHCHECK_KILL_GRACE_SECONDS" \--label t2-user-facing-health -- \bash -c "$T2_HEALTHCHECK_COMMAND" || health_status=$?if [ "$health_status" -eq 0 ]; thendiff --git a/scripts/tests/test-hcc-lifecycle-cleanup.sh b/scripts/tests/test-hcc-lifecycle-cleanup.shnew file mode 100755index 000000000..988f7bb65--- /dev/null+++ b/scripts/tests/test-hcc-lifecycle-cleanup.sh@@ -0,0 +1,200 @@+#!/usr/bin/env bash+set -euo pipefail+ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"+tmp="$(mktemp -d)"+trap 'rm -rf -- "$tmp"' EXIT+# Evaluate the production declaration and validator, without sourcing T2's+# orchestrator or contacting a cluster.+source_function() {+ local text+ text="$(sed -n "/^$2() {/,/^}/p" "$1")"+ [[ -n "$text" ]] || { echo "Missing function $2" >&2; exit 1; }+ eval "$text"+}+unset T2_HEALTHCHECK_KILL_GRACE_SECONDS+eval "$(sed -n '/^T2_HEALTHCHECK_KILL_GRACE_SECONDS=/p' "$ROOT/scripts/minikube/t2.sh")"+[[ "$T2_HEALTHCHECK_KILL_GRACE_SECONDS" = 5 ]]+source_function "$ROOT/scripts/minikube/t2.sh" validate_healthcheck_contract+T2_HEALTHCHECK_TIMEOUT_SECONDS=900 T2_PLAN_STATE=already-synced+T2_HEALTHCHECK_PENDING=false T2_HEALTHCHECK_COMMAND=''+t2_fail() { printf '%s\n' "$*" >> "$tmp/validation-errors"; }+for value in 0 -1 1.5 abc 301 999999999999999999999; do+ T2_HEALTHCHECK_KILL_GRACE_SECONDS="$value"+ if validate_healthcheck_contract; then+ echo "FAIL: accepted invalid health cleanup grace $value" >&2; exit 1+ fi+done+for value in 1 5 300; do+ T2_HEALTHCHECK_KILL_GRACE_SECONDS="$value"+ validate_healthcheck_contract+done+# Verify the validated value reaches the actual runner invocation.+source_function "$ROOT/scripts/minikube/t2.sh" run_healthcheck_if_requested+T2_HEALTHCHECK_COMMAND=true T2_DEADLINE_RUNNER="$ROOT/scripts/minikube/run-with-deadline.mjs"+T2_HEALTHCHECK_KILL_GRACE_SECONDS=300+node() { printf '%s\n' "$@" > "$tmp/runner-args"; }+t2_evidence_write() { :; }+run_healthcheck_if_requested+unset -f node+awk 'last=="--kill-grace-seconds" && $0=="300" {found=1} {last=$0} END {exit !found}' "$tmp/runner-args"+echo 'PASS: health grace defaults to 5, validates 1..300 and reaches the runner'++# Execute the actual callback and supervisor, replacing only the kubectl process.+python3 - "$ROOT" "$tmp" <<'PY'+import os+from pathlib import Path+import signal+import subprocess+import sys+import time++root, tmp = map(Path, sys.argv[1:])+source = (root / 'scripts/e2e/e2e-hcc-watch-churn-readiness.sh').read_text()+start = source.index('cleanup() {\n')+end = source.index('\n}\n', start) + 3+cleanup = source[start:end]+traps = '\n'.join(line for line in source.splitlines() if line.startswith('trap '))+assert 'trap cleanup EXIT' in traps and 'TERM' in traps+kubectl = tmp / 'kubectl-fixture.py'+kubectl.write_text('''#!/usr/bin/env python3+import os, sys, time+args = sys.argv[1:]+with open(os.environ['TRACE'], 'a') as trace:+ trace.write('kubectl ' + ' '.join(args) + '\\n')+args = args[2:] # explicit --context fixture-context+if args[:2] == ['set', 'env']:+ time.sleep(float(os.environ.get('RESTORE_DELAY', '0')))+if args[:2] == ['rollout', 'status'] and os.environ.get('FAIL_RESTORE') == '1':+ sys.exit(1)+if args[0] == 'delete' and os.environ.get('FAIL_FIXTURE') == '1':+ sys.exit(1)+''')+kubectl.chmod(0o700)+child = tmp / 'cleanup-child.sh'+child.write_text('''#!/usr/bin/env bash+set -euo pipefail+ROOT=$1+source "$ROOT/scripts/e2e/_lib/hcc-watch-churn-fixture.sh"+source "$ROOT/scripts/e2e/_lib/hcc-networkpolicy-lifecycle.sh"+source "$ROOT/scripts/e2e/_lib/hcc-watch-recovery-lock.sh"+source "$ROOT/scripts/e2e/_lib/hcc-watch-lifecycle-cleanup.sh"+E2E_HCC_POLICY_LIFECYCLE=1 E2E_KUBECONTEXT=fixture-context+HCC_PATCHED=1 HCC_SCALED_DOWN=1 ORIGINAL_REPLICAS=1+HCC_DEPLOY=fixture-hcc HCC_NS=fixture-control+FLEET_CREATED=0 PROXY_CREATED=0 PROBE_CREATED=0+NP604_CREATED=1 NP604_WATCH_PID=''+NP604_SERVER=fixture-affected NP604_CONTROL=fixture-control MCP_NS=fixture-mcp+RUN_ID=fixture HCC_GATE_LOCK_ACQUIRED=1+HCC_LOG_BUFFER="${TRACE}.buffer" READY_SERIES="${TRACE}.series"+release_hcc_watch_gate_lock() {+ echo lock-released >> "$TRACE"+ HCC_GATE_LOCK_ACQUIRED=0+}+print_hcc_watch_gate_lock_instructions() { echo lock-retained >> "$TRACE"; }+print_repair_instructions() { echo repair-required >> "$TRACE"; }+print_results() { :; }+''' + cleanup + '\n' + traps + '''+echo ready > "${TRACE}.ready"+if [[ "${DIRECT_EXIT:-}" != '' ]]; then exit "$DIRECT_EXIT"; fi+sleep 30+''')+runner = root / 'scripts/minikube/run-with-deadline.mjs'+base_env = {**os.environ, 'KUBECTL_BIN': str(kubectl)}++def run_case(name, *, mode='timeout', **extra):+ trace = tmp / (name + '.trace')+ env = {**base_env, 'TRACE': str(trace), **extra}+ args = ['node', str(runner), '--timeout-seconds', '1' if mode == 'timeout' else '30',+ '--kill-grace-seconds', '300', '--label', 'cleanup-contract', '--',+ 'bash', str(child), str(root)]+ process = subprocess.Popen(args, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)+ if mode == 'term':+ deadline = time.monotonic() + 5+ ready = Path(str(trace) + '.ready')+ while not ready.exists() and process.poll() is None and time.monotonic() < deadline:+ time.sleep(0.02)+ assert ready.exists(), 'child never became ready for TERM'+ process.send_signal(signal.SIGTERM)+ try:+ stdout, stderr = process.communicate(timeout=25)+ except subprocess.TimeoutExpired:+ # Reap the owned process group even if this test fails.+ process.send_signal(signal.SIGTERM)+ try:+ process.communicate(timeout=2)+ except subprocess.TimeoutExpired:+ process.send_signal(signal.SIGTERM)+ process.communicate(timeout=3)+ raise+ expected = 124 if mode == 'timeout' else 143 if mode == 'term' else int(extra['DIRECT_EXIT'])+ assert process.returncode == expected, (name, process.returncode, stdout, stderr)+ lines = trace.read_text().splitlines()+ restore = next(i for i, line in enumerate(lines) if ' rollout status ' in line)+ fixture = [i for i, line in enumerate(lines) if ' delete mcpserver,secret ' in line]+ if extra.get('FAIL_RESTORE') == '1':+ assert not fixture, (name, 'removed fixtures before confirming restoration', lines)+ else:+ assert fixture and restore < fixture[0], (name, 'fixture removal preceded restoration', lines)+ if extra.get('FAIL_RESTORE') == '1' or extra.get('FAIL_FIXTURE') == '1':+ assert 'lock-retained' in lines and 'lock-released' not in lines, lines+ else:+ assert 'lock-released' in lines, (name, 'cleanup did not finish', lines, stderr)++for mode in ('timeout', 'term'):+ run_case(mode, mode=mode, RESTORE_DELAY='6')+ print(f'PASS: {mode} retains its failure after real cleanup takes more than five seconds')+run_case('restore-failure', mode='exit', DIRECT_EXIT='7', FAIL_RESTORE='1')+run_case('fixture-failure', mode='exit', DIRECT_EXIT='7', FAIL_FIXTURE='1')+print('PASS: restoration precedes fixture work; restoration/fixture failures retain the lock')++# Deliberate transport stall, bounded by the production command wrapper.+trace = tmp / 'budget.trace'+env = {**base_env, 'TRACE': str(trace), 'RESTORE_DELAY': '5'}+command = '''source "$1/scripts/e2e/_lib/hcc-watch-lifecycle-cleanup.sh"+E2E_KUBECONTEXT=fixture-context+HCC_CLEANUP_PHASE_DEADLINE=$((SECONDS + 1))+hcc_cleanup_kctl set env deployment/fixture-hcc+'''+started = time.monotonic()+result = subprocess.run(['bash', '-c', command, 'budget-check', str(root)], env=env,+ capture_output=True, text=True, timeout=8)+assert result.returncode == 124, (result.returncode, result.stderr)+assert time.monotonic() - started < 4+print('PASS: actual cleanup command boundary terminates a stalled request at its budget')+PY++# Actual absence predicates must handle partial setup and resources that lack+# the suite/run labels (the proxy has an app label instead).+(+ set -euo pipefail+ source "$ROOT/scripts/e2e/_lib/hcc-networkpolicy-lifecycle.sh"+ source "$ROOT/scripts/e2e/_lib/hcc-watch-lifecycle-cleanup.sh"+ NP604_CREATED=0 FLEET_CREATED=0 PROXY_CREATED=0 PROBE_CREATED=0+ HCC_CLEANUP_HOSTS='' RUN_ID=fixture HCC_NS=fixture-control+ unset NP604_SERVER NP604_CONTROL MCP_NS+ kctl() { :; }+ hcc_lifecycle_fixture_absent+ echo 'PASS: absence check tolerates setup failure before NP604 names exist under nounset'++ PROXY_CREATED=1 PROXY_NAME=fixture-proxy+ PROXY_EGRESS_NP=fixture-egress HCC_PROXY_NP=fixture-hcc PROBE_EGRESS_NP=fixture-probe+ remaining_kind=deployment+ kctl() {+ if [[ "$remaining_kind" = deployment && "$*" = "get deployment,service $PROXY_NAME -n $HCC_NS --ignore-not-found -o name" ]]; then+ printf 'deployment.apps/%s\n' "$PROXY_NAME"+ elif [[ "$remaining_kind" = pod && "$*" = "get pod -n $HCC_NS -l app=$PROXY_NAME -o name" ]]; then+ printf 'pod/%s-terminating\n' "$PROXY_NAME"+ fi+ return 0+ }+ if hcc_lifecycle_fixture_absent; then+ echo 'FAIL: exact-name proxy resource was mistaken for absence' >&2; exit 1+ fi+ remaining_kind=pod+ if hcc_lifecycle_fixture_absent; then+ echo 'FAIL: remaining app-labelled proxy pod was mistaken for absence' >&2; exit 1+ fi+ remaining_kind=none+ hcc_lifecycle_fixture_absent+ echo 'PASS: proxy absence requires both exact-name resources and app-labelled pods to disappear'+)diff --git a/scripts/tests/test-hcc-watch-churn-empty-fleet.sh b/scripts/tests/test-hcc-watch-churn-empty-fleet.shindex cc215e336..43c8b5a6d 100755--- a/scripts/tests/test-hcc-watch-churn-empty-fleet.sh+++ b/scripts/tests/test-hcc-watch-churn-empty-fleet.sh@@ -105,27 +105,5 @@ assert_hcc_sampled_recoveries[[ ! -s "$tmp/sample-results" ]] # Disabled sampling must not produce vacuous PASS.echo 'PASS: stress sampling remains required; focused mode makes no sampled-outage claim'-(- HCC_PATCHED=1 HCC_SCALED_DOWN=1 ORIGINAL_REPLICAS=1- HCC_DEPLOY=host-context-controller HCC_NS=control-plane- FLEET_CREATED=0 PROXY_CREATED=0 PROBE_CREATED=0- E2E_HCC_POLICY_LIFECYCLE=1 NP604_CREATED=1 NP604_WATCH_PID=''- NP604_SERVER=fixture-affected NP604_CONTROL=fixture-control- PROXY_EGRESS_NP=fixture-egress HCC_PROXY_NP=fixture-hcc PROBE_EGRESS_NP=fixture-probe- HCC_LOG_BUFFER="$tmp/buffer" READY_SERIES="$tmp/series"- kctl() { printf '%s\n' "$*" >> "$tmp/order"; }- stop_hcc_recovery_log_stream() { :; }- hcc_pods_absent() { :; }- wait_until() { shift 2; "$@"; }- restore_hcc_after_churn() { :; }- finalize_hcc_watch_gate_lock() { :; }- print_results() { :; }- eval "$cleanup_source"- cleanup-)-awk '- /get deployment,service,networkpolicy/ && !owner_check {owner_check=NR}- /scale deployment host-context-controller.*--replicas=0/ && !stop {stop=NR}- END {exit !(owner_check && stop && owner_check < stop)}-' "$tmp/order"-echo 'PASS: actual cleanup checks owner-driven fixture removal before stopping HCC'+# Lifecycle timeout, restore-first ordering and retained-lock cases execute the+# real cleanup and supervisor in test-hcc-lifecycle-cleanup.sh.