Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(hcc): address adversarial audit findings

Alfredo Lopezce2fdd83a8cf
diff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.json
index 7b7905790..752839af1 100644
--- a/host-context-controller/package-lock.json
+++ b/host-context-controller/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.312",
+ "version": "0.1.313",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-host-context-controller",
- "version": "0.1.312",
+ "version": "0.1.313",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@clerum/codex-catalog-projection": "file:../packages/codex-catalog-projection",
diff --git a/host-context-controller/package.json b/host-context-controller/package.json
index eff44cdda..5c8e6e083 100644
--- a/host-context-controller/package.json
+++ b/host-context-controller/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.312",
+ "version": "0.1.313",
"description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API",
"main": "dist/main.js",
"engines": {
diff --git a/host-context-controller/src/k8sClient.test.ts b/host-context-controller/src/k8sClient.test.ts
index 23c963224..a0c121cb7 100644
--- a/host-context-controller/src/k8sClient.test.ts
+++ b/host-context-controller/src/k8sClient.test.ts
@@ -14,7 +14,10 @@ import {
listAllSharedFileSystems,
} from './k8sClient'
import { registry } from './metrics'
-import { DESIRED_NETWORKPOLICY_INVENTORY_CHANGED_MESSAGE } from './networkPolicyReconciler'
+import {
+ DESIRED_NETWORKPOLICY_INVENTORY_CHANGED_MESSAGE,
+ type NetworkPolicyReconciler,
+} from './networkPolicyReconciler'
import { ContextMapperServer } from './server'
import type { HostCRD, McpServerCRD } from './types'
@@ -11015,6 +11018,11 @@ describe('McpServerWatcher readiness under sustained watch churn (GKE Premature-
expect(full).toHaveBeenCalledTimes(1)
expect(scoped).toHaveBeenCalledTimes(1)
expect(retry).toHaveBeenCalledTimes(1)
+ expect(scoped).toHaveBeenLastCalledWith(
+ state.servers.get(server.name),
+ state.mcpWatchGeneration
+ )
+ expect(retry).toHaveBeenLastCalledWith('MODIFIED', state.servers.get(server.name))
for (let repeat = 0; repeat < 3; repeat++) {
await callback(
'MODIFIED',
@@ -11027,9 +11035,82 @@ describe('McpServerWatcher readiness under sustained watch churn (GKE Premature-
expect(full).toHaveBeenCalledTimes(2)
expect(scoped).toHaveBeenCalledTimes(2)
expect(retry).toHaveBeenCalledTimes(2)
+ expect(scoped).toHaveBeenLastCalledWith(
+ state.servers.get(server.name),
+ state.mcpWatchGeneration
+ )
+ expect(retry).toHaveBeenLastCalledWith('MODIFIED', state.servers.get(server.name))
await watcher.stop()
})
+ it.each(['current', 'verdict changed', 'watch retired'])(
+ 'retries incomplete Context work only while its server effect remains %s (#604)',
+ async disposition => {
+ const watcher = new McpServerWatcher()
+ const state = watcher as any
+ markMcpServerInventoryAuthoritative(watcher)
+ state.contextCacheSynced = true
+ const server: McpServerCRD = {
+ name: 'incomplete-policy-server',
+ namespace: 'mcp-server',
+ uid: 'incomplete-policy-uid',
+ generation: 1,
+ spec: {
+ image: 'fixture:v1',
+ contextRef: 'ctx',
+ transport: { type: 'streamableHttp', port: 3000 },
+ envSecret: { name: 'fixture-env', keys: [] },
+ },
+ }
+ const context = {
+ name: 'linked-context',
+ namespace: 'mcp-server',
+ spec: { contextId: 'ctx', mcpServers: [server.name] },
+ }
+ state.servers.set(server.name, server)
+ state.contexts.set(context.name, context)
+ const entered = deferred()
+ const release = deferred<boolean>()
+ let admittedIsCurrent: (() => boolean) | undefined
+ const reconcile = vi
+ .spyOn(state.netPolReconciler as NetworkPolicyReconciler, 'reconcileContext')
+ .mockImplementation(async (_context, options) => {
+ admittedIsCurrent = options?.isCurrent
+ entered.resolve()
+ return release.promise
+ })
+ const retry = vi.spyOn(state, 'scheduleExternalEgressRetry').mockImplementation(() => {})
+ try {
+ // Keep the actual Context queue and helper; pause only at the policy API.
+ const pending = state.reconcileContextsForMcpServer(server, state.mcpWatchGeneration)
+ await entered.promise
+ expect(reconcile).toHaveBeenCalledWith(context, { isCurrent: expect.any(Function) })
+ expect(admittedIsCurrent?.()).toBe(true)
+ if (disposition === 'verdict changed') {
+ state.servers.set(server.name, {
+ ...server,
+ status: {
+ conditions: [{ type: 'SecretResolved', status: 'False', reason: 'SecretNotFound' }],
+ },
+ })
+ } else if (disposition === 'watch retired') {
+ state.mcpWatchGeneration += 1
+ }
+ expect(admittedIsCurrent?.()).toBe(disposition === 'current')
+ release.resolve(false)
+ await pending
+ if (disposition === 'current') {
+ expect(retry).toHaveBeenCalledExactlyOnceWith('MODIFIED', server)
+ } else {
+ expect(retry).not.toHaveBeenCalled()
+ }
+ } finally {
+ release.resolve(false)
+ await watcher.stop()
+ }
+ }
+ )
+
it('installHostSnapshot bumps the desired revision only when re-listed content changed', () => {
const watcher = new McpServerWatcher()
const hostA = {
diff --git a/host-context-controller/src/mcpServerSafety.ts b/host-context-controller/src/mcpServerSafety.ts
index bcdde5a58..633f6b87d 100644
--- a/host-context-controller/src/mcpServerSafety.ts
+++ b/host-context-controller/src/mcpServerSafety.ts
@@ -103,6 +103,9 @@ export function shouldFailClosedForSecretFailure(reason: string | undefined): bo
* HCC owns policies for both runtime owners, but its Secret verdict only
* controls HCC-managed runtime. WRC permits optional keys HCC cannot judge.
* A removed reference makes an old SecretResolved condition irrelevant.
+ * Before the first published verdict, policy intent is optimistic: allows may
+ * exist while the runtime owner validates the Secret before creating runtime
+ * resources. A published failure then revokes those allows through convergence.
*/
export function runtimeDesired(server: McpServerCRD): boolean {
if (server.spec.enabled === false) return false
diff --git a/host-context-controller/src/networkPolicyReconciler.test.ts b/host-context-controller/src/networkPolicyReconciler.test.ts
index 4f8ba0372..d029f9d70 100644
--- a/host-context-controller/src/networkPolicyReconciler.test.ts
+++ b/host-context-controller/src/networkPolicyReconciler.test.ts
@@ -6413,8 +6413,28 @@ describe('NetworkPolicyReconciler', () => {
if (reason === 'Disabled') failed.spec.enabled = false
else failed.status = { conditions: [{ type: 'SecretResolved', status: 'False', reason }] }
f.cache.set(failed.name, failed)
+ mockCustomApi.patchNamespacedCustomObjectStatus.mockClear()
await f.run()
expect(f.store.size).toBe(0)
+ const statusPatch = mockCustomApi.patchNamespacedCustomObjectStatus.mock.lastCall?.[0]
+ expect(statusPatch).toMatchObject({ name: failed.name, namespace: failed.namespace })
+ const conditionPatch = statusPatch?.body.find(
+ (operation: { path: string }) =>
+ operation.path === '/status' || operation.path === '/status/conditions'
+ )
+ const publishedConditions =
+ conditionPatch?.path === '/status'
+ ? conditionPatch.value.conditions
+ : conditionPatch?.value
+ expect(publishedConditions).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({
+ type: 'ExternalEgressReady',
+ status: 'False',
+ reason: 'RuntimeNotDesired',
+ }),
+ ])
+ )
expect(f.events.filter(event => event.startsWith('delete:'))).toHaveLength(4)
const settled = f.events.length
for (let cycle = 0; cycle < 3; cycle++) await f.run()
diff --git a/scripts/e2e/_lib/hcc-networkpolicy-lifecycle.sh b/scripts/e2e/_lib/hcc-networkpolicy-lifecycle.sh
index 106a06455..6cc161d10 100755
--- a/scripts/e2e/_lib/hcc-networkpolicy-lifecycle.sh
+++ b/scripts/e2e/_lib/hcc-networkpolicy-lifecycle.sh
@@ -182,13 +182,10 @@ np604_cleanup() {
fi
NP604_WATCH_PID=''
[ "$NP604_CREATED" = 1 ] || return 0
+ # Request only: the lifecycle cleanup owner verifies absence of the fixture
+ # and its children under the shared cleanup deadline after restoring HCC.
np604_kctl delete mcpserver,secret -n "$MCP_NS" \
- -l "e2e.clerum.io/suite=hcc-np604,e2e.clerum.io/run=${RUN_ID}" --ignore-not-found >/dev/null || return 1
- # Consume the deletion events while the owning controller is still running.
- # Removing the Context only after stopping HCC would leave its allows to a
- # namespace-wide orphan sweep, whose safety cap correctly refuses this fleet.
- [ "${HCC_CLEANUP_NONBLOCKING:-false}" = true ] && return 0
- wait_until 120 'NP604 owner-driven cleanup before HCC stop' np604_resources_absent
+ -l "e2e.clerum.io/suite=hcc-np604,e2e.clerum.io/run=${RUN_ID}" --ignore-not-found --wait=false >/dev/null
}
np604_resources_absent() {
diff --git a/scripts/e2e/_lib/hcc-watch-lifecycle-cleanup.sh b/scripts/e2e/_lib/hcc-watch-lifecycle-cleanup.sh
index dd92d96e4..a94799e7a 100755
--- a/scripts/e2e/_lib/hcc-watch-lifecycle-cleanup.sh
+++ b/scripts/e2e/_lib/hcc-watch-lifecycle-cleanup.sh
@@ -93,9 +93,6 @@ hcc_lifecycle_fixture_absent() {
cleanup_hcc_lifecycle() (
local status=$1 cleanup_failed=0 restore_ok=1 started=$SECONDS
local HCC_CLEANUP_PHASE_DEADLINE=$((SECONDS + 150)) HCC_CLEANUP_HOSTS=''
- # Consumed through Bash dynamic scope by np604_cleanup in the sourced helper.
- # shellcheck disable=SC2034
- local HCC_CLEANUP_NONBLOCKING=true
trap - EXIT
trap ':' TERM INT HUP QUIT
set +e
diff --git a/scripts/e2e/e2e-hcc-networkpolicy-lifecycle.sh b/scripts/e2e/e2e-hcc-networkpolicy-lifecycle.sh
index 423043cf5..168b4c4c0 100755
--- a/scripts/e2e/e2e-hcc-networkpolicy-lifecycle.sh
+++ b/scripts/e2e/e2e-hcc-networkpolicy-lifecycle.sh
@@ -9,6 +9,13 @@ SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
echo 'FAIL: profile and context must match' >&2
exit 2
}
+# Restoration owns 270 seconds plus a 30-second supervisor margin. T2 exports
+# its validated actual runner grace; refuse direct/default-grace entry before
+# enabling fault injection. The runner's maximum grace is also 300 seconds.
+[[ "${T2_HEALTHCHECK_KILL_GRACE_SECONDS:-}" = 300 ]] || {
+ echo 'FAIL: lifecycle requires T2_HEALTHCHECK_KILL_GRACE_SECONDS=300 (270s cleanup + 30s margin); use make minikube-t2-hcc-networkpolicy-lifecycle' >&2
+ exit 2
+}
export KUBECONTEXT="$CONTROL_API_REAL_PG_CONTEXT"
export E2E_HCC_WATCH_FAULT_INJECTION=1
export E2E_HCC_POLICY_LIFECYCLE=1
diff --git a/scripts/e2e/e2e-hcc-watch-churn-readiness.sh b/scripts/e2e/e2e-hcc-watch-churn-readiness.sh
index 4d314323b..0dbc5f6fa 100755
--- a/scripts/e2e/e2e-hcc-watch-churn-readiness.sh
+++ b/scripts/e2e/e2e-hcc-watch-churn-readiness.sh
@@ -301,6 +301,12 @@ assert_hcc_sampled_recoveries() {
cleanup() {
local status=$? cleanup_failed=0 restore_ok=1
+ if [ "${E2E_HCC_POLICY_LIFECYCLE:-0}" = 1 ]; then
+ # Ignore cancellation across the handoff: unlike a caught handler, this
+ # disposition survives subshell creation until cleanup installs its traps.
+ # The supervisor still escalates repeated cancellation to SIGKILL.
+ trap '' TERM INT HUP QUIT
+ fi
trap - EXIT
set +e
if [ "${E2E_HCC_POLICY_LIFECYCLE:-0}" = 1 ]; then
diff --git a/scripts/minikube/t2.sh b/scripts/minikube/t2.sh
index 7ac29d490..365afc806 100755
--- a/scripts/minikube/t2.sh
+++ b/scripts/minikube/t2.sh
@@ -395,6 +395,7 @@ run_healthcheck_if_requested() {
t2_evidence_write Health NOT_RUN 'no profile-owned user-facing health command was supplied'
return 0
fi
+ T2_HEALTHCHECK_KILL_GRACE_SECONDS="$T2_HEALTHCHECK_KILL_GRACE_SECONDS" \
node "$T2_DEADLINE_RUNNER" \
--timeout-seconds "$T2_HEALTHCHECK_TIMEOUT_SECONDS" \
--heartbeat-seconds 20 --kill-grace-seconds "$T2_HEALTHCHECK_KILL_GRACE_SECONDS" \
diff --git a/scripts/tests/test-hcc-lifecycle-cleanup.sh b/scripts/tests/test-hcc-lifecycle-cleanup.sh
index 988f7bb65..b3e82d742 100755
--- a/scripts/tests/test-hcc-lifecycle-cleanup.sh
+++ b/scripts/tests/test-hcc-lifecycle-cleanup.sh
@@ -37,7 +37,53 @@ t2_evidence_write() { :; }
run_healthcheck_if_requested
unset -f node
awk 'last=="--kill-grace-seconds" && $0=="300" {found=1} {last=$0} END {exit !found}' "$tmp/runner-args"
-echo 'PASS: health grace defaults to 5, validates 1..300 and reaches the runner'
+# The declaration is normally a shell variable, not inherited by child scripts.
+# Exercise the actual runner and child shell, not only its argument list.
+export -n T2_HEALTHCHECK_KILL_GRACE_SECONDS
+T2_HEALTHCHECK_COMMAND='test "$T2_HEALTHCHECK_KILL_GRACE_SECONDS" = 300'
+run_healthcheck_if_requested
+echo 'PASS: health grace defaults to 5, validates 1..300 and reaches the runner and child'
+
+# Intercept only the wrapper's final exec. Invalid grace must fail before it
+# can enter the mutating gate, and the canonical 300-second target must enter.
+mkdir -p "$tmp/wrapper-bin"
+cat > "$tmp/wrapper-bin/bash" <<'SH'
+#!/bin/bash
+printf '%s\n' "$*" > "$WRAPPER_GATE_ENTERED"
+SH
+chmod +x "$tmp/wrapper-bin/bash"
+for value in missing 5 269 270 299 invalid 301; do
+ rm -f "$tmp/wrapper-entered"
+ (
+ unset T2_HEALTHCHECK_KILL_GRACE_SECONDS
+ if [ "$value" != missing ]; then export T2_HEALTHCHECK_KILL_GRACE_SECONDS="$value"; fi
+ PATH="$tmp/wrapper-bin:$PATH" WRAPPER_GATE_ENTERED="$tmp/wrapper-entered" \
+ MINIKUBE_PROFILE=fixture-context CONTROL_API_REAL_PG_CONTEXT=fixture-context \
+ /bin/bash "$ROOT/scripts/e2e/e2e-hcc-networkpolicy-lifecycle.sh"
+ ) > "$tmp/wrapper-output" 2>&1 && {
+ echo "FAIL: lifecycle wrapper accepted grace $value" >&2; exit 1
+ }
+ [[ ! -e "$tmp/wrapper-entered" ]]
+ grep -q 'requires T2_HEALTHCHECK_KILL_GRACE_SECONDS=300' "$tmp/wrapper-output"
+done
+PATH="$tmp/wrapper-bin:$PATH" WRAPPER_GATE_ENTERED="$tmp/wrapper-entered" \
+ MINIKUBE_PROFILE=fixture-context CONTROL_API_REAL_PG_CONTEXT=fixture-context \
+ T2_HEALTHCHECK_KILL_GRACE_SECONDS=300 \
+ /bin/bash "$ROOT/scripts/e2e/e2e-hcc-networkpolicy-lifecycle.sh"
+grep -q 'e2e-hcc-watch-churn-readiness.sh' "$tmp/wrapper-entered"
+echo 'PASS: lifecycle rejects missing/insufficient grace before gate entry and accepts 300'
+
+# The fixture helper submits deletion only; the lifecycle cleanup owns the
+# shared-budget absence proof. A private owner wait must not return here.
+(
+ source "$ROOT/scripts/e2e/_lib/hcc-networkpolicy-lifecycle.sh"
+ NP604_CREATED=1 MCP_NS=fixture-mcp RUN_ID=fixture
+ np604_kctl() { printf '%s\n' "$*" > "$tmp/fixture-delete"; }
+ wait_until() { echo 'FAIL: fixture cleanup started its own wait' >&2; return 1; }
+ np604_cleanup
+ grep -q -- '--wait=false' "$tmp/fixture-delete"
+)
+echo 'PASS: fixture cleanup requests deletion without a separate owner wait'
# Execute the actual callback and supervisor, replacing only the kubectl process.
python3 - "$ROOT" "$tmp" <<'PY'
@@ -94,6 +140,18 @@ print_hcc_watch_gate_lock_instructions() { echo lock-retained >> "$TRACE"; }
print_repair_instructions() { echo repair-required >> "$TRACE"; }
print_results() { :; }
''' + cleanup + '\n' + traps + '''
+if [[ "${SIGNAL_AT_CLEANUP_HANDOFF:-0}" = 1 ]]; then
+ # Probe parent and helper startup separately. Calling the real cleanup
+ # explicitly makes DEBUG observable even where Bash suppresses it in EXIT.
+ set -T
+ HANDOFF_ARMED=1
+ HANDOFF_PARENT=$BASHPID
+ trap 'if [[ "$HANDOFF_ARMED" = 1 && "$BASH_COMMAND" = "trap - EXIT" ]] && { [[ "$HANDOFF_TARGET" = parent && "$BASHPID" = "$HANDOFF_PARENT" ]] || [[ "$HANDOFF_TARGET" = helper && "$BASHPID" != "$HANDOFF_PARENT" ]]; }; then HANDOFF_ARMED=0; echo "handoff-injected:$HANDOFF_TARGET" >> "$TRACE"; kill -TERM "$BASHPID"; fi' DEBUG
+ set +e
+ status_before_cleanup() { return "$DIRECT_EXIT"; }
+ status_before_cleanup
+ cleanup
+fi
echo ready > "${TRACE}.ready"
if [[ "${DIRECT_EXIT:-}" != '' ]]; then exit "$DIRECT_EXIT"; fi
sleep 30
@@ -129,6 +187,8 @@ def run_case(name, *, mode='timeout', **extra):
expected = 124 if mode == 'timeout' else 143 if mode == 'term' else int(extra['DIRECT_EXIT'])
assert process.returncode == expected, (name, process.returncode, stdout, stderr)
lines = trace.read_text().splitlines()
+ if extra.get('SIGNAL_AT_CLEANUP_HANDOFF') == '1':
+ assert 'handoff-injected:' + extra['HANDOFF_TARGET'] in lines, (name, lines)
restore = next(i for i, line in enumerate(lines) if ' rollout status ' in line)
fixture = [i for i, line in enumerate(lines) if ' delete mcpserver,secret ' in line]
if extra.get('FAIL_RESTORE') == '1':
@@ -146,6 +206,10 @@ for mode in ('timeout', 'term'):
run_case('restore-failure', mode='exit', DIRECT_EXIT='7', FAIL_RESTORE='1')
run_case('fixture-failure', mode='exit', DIRECT_EXIT='7', FAIL_FIXTURE='1')
print('PASS: restoration precedes fixture work; restoration/fixture failures retain the lock')
+for target in ('parent', 'helper'):
+ run_case('handoff-term-' + target, mode='exit', DIRECT_EXIT='7',
+ SIGNAL_AT_CLEANUP_HANDOFF='1', HANDOFF_TARGET=target)
+print('PASS: TERM at both parent handoff and helper startup preserves restoration and original status')
# Deliberate transport stall, bounded by the production command wrapper.
trace = tmp / 'budget.trace'