Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
12.6 KB
Raw Blame
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"
tmp="$(mktemp -d)"
trap 'rm -rf -- "$tmp"' EXIT
# Evaluate the production declaration and validator, without sourcing T2's
# orchestrator or contacting a cluster.
source_function() {
  local text
  text="$(sed -n "/^$2() {/,/^}/p" "$1")"
  [[ -n "$text" ]] || { echo "Missing function $2" >&2; exit 1; }
  eval "$text"
}
unset T2_HEALTHCHECK_KILL_GRACE_SECONDS
eval "$(sed -n '/^T2_HEALTHCHECK_KILL_GRACE_SECONDS=/p' "$ROOT/scripts/minikube/t2.sh")"
[[ "$T2_HEALTHCHECK_KILL_GRACE_SECONDS" = 5 ]]
source_function "$ROOT/scripts/minikube/t2.sh" validate_healthcheck_contract
T2_HEALTHCHECK_TIMEOUT_SECONDS=900 T2_PLAN_STATE=already-synced
T2_HEALTHCHECK_PENDING=false T2_HEALTHCHECK_COMMAND=''
t2_fail() { printf '%s\n' "$*" >> "$tmp/validation-errors"; }
for value in 0 -1 1.5 abc 301 999999999999999999999; do
  T2_HEALTHCHECK_KILL_GRACE_SECONDS="$value"
  if validate_healthcheck_contract; then
    echo "FAIL: accepted invalid health cleanup grace $value" >&2; exit 1
  fi
done
for value in 1 5 300; do
  T2_HEALTHCHECK_KILL_GRACE_SECONDS="$value"
  validate_healthcheck_contract
done
# Verify the validated value reaches the actual runner invocation.
source_function "$ROOT/scripts/minikube/t2.sh" run_healthcheck_if_requested
# The isolated invocation forwards a parent lease identity but does not enter
# the orchestrator or validate a live lease. Keep its synthetic identity local.
T2_LOCK_TOKEN=fixture-lease T2_PROFILE=fixture-context T2_CONTEXT=fixture-context
T2_PROJECT_DIR="$tmp" T2_LOCK_ROOT="$tmp/locks"
T2_HEALTHCHECK_COMMAND=true T2_DEADLINE_RUNNER="$ROOT/scripts/minikube/run-with-deadline.mjs"
T2_HEALTHCHECK_KILL_GRACE_SECONDS=300
node() { printf '%s\n' "$@" > "$tmp/runner-args"; }
t2_evidence_write() { :; }
run_healthcheck_if_requested
unset -f node
awk 'last=="--kill-grace-seconds" && $0=="300" {found=1} {last=$0} END {exit !found}' "$tmp/runner-args"
# The declaration is normally a shell variable, not inherited by child scripts.
# Exercise the actual runner and child shell, not only its argument list.
export -n T2_HEALTHCHECK_KILL_GRACE_SECONDS
T2_HEALTHCHECK_COMMAND='test "$T2_HEALTHCHECK_KILL_GRACE_SECONDS" = 300'
run_healthcheck_if_requested
echo 'PASS: health grace defaults to 5, validates 1..300 and reaches the runner and child'

# Intercept only the wrapper's final exec. Invalid grace must fail before it
# can enter the mutating gate, and the canonical 300-second target must enter.
mkdir -p "$tmp/wrapper-bin"
cat > "$tmp/wrapper-bin/bash" <<'SH'
#!/bin/bash
printf '%s\n' "$*" > "$WRAPPER_GATE_ENTERED"
SH
chmod +x "$tmp/wrapper-bin/bash"
for value in missing 5 269 270 299 invalid 301; do
  rm -f "$tmp/wrapper-entered"
  (
    unset T2_HEALTHCHECK_KILL_GRACE_SECONDS
    if [ "$value" != missing ]; then export T2_HEALTHCHECK_KILL_GRACE_SECONDS="$value"; fi
    PATH="$tmp/wrapper-bin:$PATH" WRAPPER_GATE_ENTERED="$tmp/wrapper-entered" \
      MINIKUBE_PROFILE=fixture-context CONTROL_API_REAL_PG_CONTEXT=fixture-context \
      /bin/bash "$ROOT/scripts/e2e/e2e-hcc-networkpolicy-lifecycle.sh"
  ) > "$tmp/wrapper-output" 2>&1 && {
    echo "FAIL: lifecycle wrapper accepted grace $value" >&2; exit 1
  }
  [[ ! -e "$tmp/wrapper-entered" ]]
  grep -q 'requires T2_HEALTHCHECK_KILL_GRACE_SECONDS=300' "$tmp/wrapper-output"
done
PATH="$tmp/wrapper-bin:$PATH" WRAPPER_GATE_ENTERED="$tmp/wrapper-entered" \
  MINIKUBE_PROFILE=fixture-context CONTROL_API_REAL_PG_CONTEXT=fixture-context \
  T2_HEALTHCHECK_KILL_GRACE_SECONDS=300 \
  /bin/bash "$ROOT/scripts/e2e/e2e-hcc-networkpolicy-lifecycle.sh"
grep -q 'e2e-hcc-watch-churn-readiness.sh' "$tmp/wrapper-entered"
echo 'PASS: lifecycle rejects missing/insufficient grace before gate entry and accepts 300'

# The fixture helper submits deletion only; the lifecycle cleanup owns the
# shared-budget absence proof. A private owner wait must not return here.
(
  source "$ROOT/scripts/e2e/_lib/hcc-networkpolicy-lifecycle.sh"
  NP604_CREATED=1 MCP_NS=fixture-mcp RUN_ID=fixture
  np604_kctl() { printf '%s\n' "$*" > "$tmp/fixture-delete"; }
  wait_until() { echo 'FAIL: fixture cleanup started its own wait' >&2; return 1; }
  np604_cleanup
  grep -q -- '--wait=false' "$tmp/fixture-delete"
)
echo 'PASS: fixture cleanup requests deletion without a separate owner wait'

# Execute the actual callback and supervisor, replacing only the kubectl process.
python3 - "$ROOT" "$tmp" <<'PY'
import os
from pathlib import Path
import signal
import subprocess
import sys
import time

root, tmp = map(Path, sys.argv[1:])
source = (root / 'scripts/e2e/e2e-hcc-watch-churn-readiness.sh').read_text()
start = source.index('cleanup() {\n')
end = source.index('\n}\n', start) + 3
cleanup = source[start:end]
traps = '\n'.join(line for line in source.splitlines() if line.startswith('trap '))
assert 'trap cleanup EXIT' in traps and 'TERM' in traps
kubectl = tmp / 'kubectl-fixture.py'
kubectl.write_text('''#!/usr/bin/env python3
import os, sys, time
args = sys.argv[1:]
with open(os.environ['TRACE'], 'a') as trace:
    trace.write('kubectl ' + ' '.join(args) + '\\n')
args = args[2:] # explicit --context fixture-context
if args[:2] == ['set', 'env']:
    time.sleep(float(os.environ.get('RESTORE_DELAY', '0')))
if args[:2] == ['rollout', 'status'] and os.environ.get('FAIL_RESTORE') == '1':
    sys.exit(1)
if args[0] == 'delete' and os.environ.get('FAIL_FIXTURE') == '1':
    sys.exit(1)
''')
kubectl.chmod(0o700)
child = tmp / 'cleanup-child.sh'
child.write_text('''#!/usr/bin/env bash
set -euo pipefail
ROOT=$1
source "$ROOT/scripts/e2e/_lib/hcc-watch-churn-fixture.sh"
source "$ROOT/scripts/e2e/_lib/hcc-networkpolicy-lifecycle.sh"
source "$ROOT/scripts/e2e/_lib/hcc-watch-recovery-lock.sh"
source "$ROOT/scripts/e2e/_lib/hcc-watch-lifecycle-cleanup.sh"
E2E_HCC_POLICY_LIFECYCLE=1 E2E_KUBECONTEXT=fixture-context
HCC_PATCHED=1 HCC_SCALED_DOWN=1 ORIGINAL_REPLICAS=1
HCC_DEPLOY=fixture-hcc HCC_NS=fixture-control
FLEET_CREATED=0 PROXY_CREATED=0 PROBE_CREATED=0
NP604_CREATED=1 NP604_WATCH_PID=''
NP604_SERVER=fixture-affected NP604_CONTROL=fixture-control MCP_NS=fixture-mcp
RUN_ID=fixture HCC_GATE_LOCK_ACQUIRED=1
HCC_LOG_BUFFER="${TRACE}.buffer" READY_SERIES="${TRACE}.series"
release_hcc_watch_gate_lock() {
  echo lock-released >> "$TRACE"
  HCC_GATE_LOCK_ACQUIRED=0
}
print_hcc_watch_gate_lock_instructions() { echo lock-retained >> "$TRACE"; }
print_repair_instructions() { echo repair-required >> "$TRACE"; }
print_results() { :; }
''' + cleanup + '\n' + traps + '''
if [[ "${SIGNAL_AT_CLEANUP_HANDOFF:-0}" = 1 ]]; then
  # Probe parent and helper startup separately. Calling the real cleanup
  # explicitly makes DEBUG observable even where Bash suppresses it in EXIT.
  set -T
  HANDOFF_ARMED=1
  # BASH_SUBSHELL also exists in macOS Bash 3.2. Signal the current shell
  # via a direct child so the helper receives TERM, not the top-level $$ PID.
  HANDOFF_PARENT=$BASH_SUBSHELL
  trap 'if [[ "$HANDOFF_ARMED" = 1 && "$BASH_COMMAND" = "trap - EXIT" ]] && { [[ "$HANDOFF_TARGET" = parent && "$BASH_SUBSHELL" = "$HANDOFF_PARENT" ]] || [[ "$HANDOFF_TARGET" = helper && "$BASH_SUBSHELL" != "$HANDOFF_PARENT" ]]; }; then HANDOFF_ARMED=0; echo "handoff-injected:$HANDOFF_TARGET" >> "$TRACE"; python3 -c "import os, signal; os.kill(os.getppid(), signal.SIGTERM)"; fi' DEBUG
  set +e
  status_before_cleanup() { return "$DIRECT_EXIT"; }
  status_before_cleanup
  cleanup
fi
echo ready > "${TRACE}.ready"
if [[ "${DIRECT_EXIT:-}" != '' ]]; then exit "$DIRECT_EXIT"; fi
sleep 30
''')
runner = root / 'scripts/minikube/run-with-deadline.mjs'
base_env = {**os.environ, 'KUBECTL_BIN': str(kubectl)}

def run_case(name, *, mode='timeout', **extra):
    trace = tmp / (name + '.trace')
    env = {**base_env, 'TRACE': str(trace), **extra}
    args = ['node', str(runner), '--timeout-seconds', '1' if mode == 'timeout' else '30',
            '--kill-grace-seconds', '300', '--label', 'cleanup-contract', '--',
            'bash', str(child), str(root)]
    process = subprocess.Popen(args, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
    if mode == 'term':
        deadline = time.monotonic() + 5
        ready = Path(str(trace) + '.ready')
        while not ready.exists() and process.poll() is None and time.monotonic() < deadline:
            time.sleep(0.02)
        assert ready.exists(), 'child never became ready for TERM'
        process.send_signal(signal.SIGTERM)
    try:
        stdout, stderr = process.communicate(timeout=25)
    except subprocess.TimeoutExpired:
        # Reap the owned process group even if this test fails.
        process.send_signal(signal.SIGTERM)
        try:
            process.communicate(timeout=2)
        except subprocess.TimeoutExpired:
            process.send_signal(signal.SIGTERM)
            process.communicate(timeout=3)
        raise
    expected = 124 if mode == 'timeout' else 143 if mode == 'term' else int(extra['DIRECT_EXIT'])
    assert process.returncode == expected, (name, process.returncode, stdout, stderr)
    lines = trace.read_text().splitlines()
    if extra.get('SIGNAL_AT_CLEANUP_HANDOFF') == '1':
        assert 'handoff-injected:' + extra['HANDOFF_TARGET'] in lines, (name, lines)
    restore = next(i for i, line in enumerate(lines) if ' rollout status ' in line)
    fixture = [i for i, line in enumerate(lines) if ' delete mcpserver,secret ' in line]
    if extra.get('FAIL_RESTORE') == '1':
        assert not fixture, (name, 'removed fixtures before confirming restoration', lines)
    else:
        assert fixture and restore < fixture[0], (name, 'fixture removal preceded restoration', lines)
    if extra.get('FAIL_RESTORE') == '1' or extra.get('FAIL_FIXTURE') == '1':
        assert 'lock-retained' in lines and 'lock-released' not in lines, lines
    else:
        assert 'lock-released' in lines, (name, 'cleanup did not finish', lines, stderr)

for mode in ('timeout', 'term'):
    run_case(mode, mode=mode, RESTORE_DELAY='6')
    print(f'PASS: {mode} retains its failure after real cleanup takes more than five seconds')
run_case('restore-failure', mode='exit', DIRECT_EXIT='7', FAIL_RESTORE='1')
run_case('fixture-failure', mode='exit', DIRECT_EXIT='7', FAIL_FIXTURE='1')
print('PASS: restoration precedes fixture work; restoration/fixture failures retain the lock')
for target in ('parent', 'helper'):
    run_case('handoff-term-' + target, mode='exit', DIRECT_EXIT='7',
             SIGNAL_AT_CLEANUP_HANDOFF='1', HANDOFF_TARGET=target)
print('PASS: TERM at both parent handoff and helper startup preserves restoration and original status')

# Deliberate transport stall, bounded by the production command wrapper.
trace = tmp / 'budget.trace'
env = {**base_env, 'TRACE': str(trace), 'RESTORE_DELAY': '5'}
command = '''source "$1/scripts/e2e/_lib/hcc-watch-lifecycle-cleanup.sh"
E2E_KUBECONTEXT=fixture-context
HCC_CLEANUP_PHASE_DEADLINE=$((SECONDS + 1))
hcc_cleanup_kctl set env deployment/fixture-hcc
'''
started = time.monotonic()
result = subprocess.run(['bash', '-c', command, 'budget-check', str(root)], env=env,
                        capture_output=True, text=True, timeout=8)
assert result.returncode == 124, (result.returncode, result.stderr)
assert time.monotonic() - started < 4
print('PASS: actual cleanup command boundary terminates a stalled request at its budget')
PY

# Actual absence predicates must handle partial setup and resources that lack
# the suite/run labels (the proxy has an app label instead).
(
  set -euo pipefail
  source "$ROOT/scripts/e2e/_lib/hcc-networkpolicy-lifecycle.sh"
  source "$ROOT/scripts/e2e/_lib/hcc-watch-lifecycle-cleanup.sh"
  NP604_CREATED=0 FLEET_CREATED=0 PROXY_CREATED=0 PROBE_CREATED=0
  HCC_CLEANUP_HOSTS='' RUN_ID=fixture HCC_NS=fixture-control
  unset NP604_SERVER NP604_CONTROL MCP_NS
  kctl() { :; }
  hcc_lifecycle_fixture_absent
  echo 'PASS: absence check tolerates setup failure before NP604 names exist under nounset'

  PROXY_CREATED=1 PROXY_NAME=fixture-proxy
  PROXY_EGRESS_NP=fixture-egress HCC_PROXY_NP=fixture-hcc PROBE_EGRESS_NP=fixture-probe
  remaining_kind=deployment
  kctl() {
    if [[ "$remaining_kind" = deployment && "$*" = "get deployment,service $PROXY_NAME -n $HCC_NS --ignore-not-found -o name" ]]; then
      printf 'deployment.apps/%s\n' "$PROXY_NAME"
    elif [[ "$remaining_kind" = pod && "$*" = "get pod -n $HCC_NS -l app=$PROXY_NAME -o name" ]]; then
      printf 'pod/%s-terminating\n' "$PROXY_NAME"
    fi
    return 0
  }
  if hcc_lifecycle_fixture_absent; then
    echo 'FAIL: exact-name proxy resource was mistaken for absence' >&2; exit 1
  fi
  remaining_kind=pod
  if hcc_lifecycle_fixture_absent; then
    echo 'FAIL: remaining app-labelled proxy pod was mistaken for absence' >&2; exit 1
  fi
  remaining_kind=none
  hcc_lifecycle_fixture_absent
  echo 'PASS: proxy absence requires both exact-name resources and app-labelled pods to disappear'
)