Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
7.5 KB
Raw Blame
#!/usr/bin/env bash

HCC_CLEANUP_COMMAND_RUNNER="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)/hcc-cleanup-command.mjs"

# Each request consumes the remaining phase budget, including transport stalls.
# Never start a nested detached deadline runner in the cleanup process group.
hcc_cleanup_kctl() {
  local remaining request_seconds command_seconds arg is_delete=false is_rollout=false
  remaining=$((HCC_CLEANUP_PHASE_DEADLINE - SECONDS))
  [ "$remaining" -gt 0 ] || return 124
  request_seconds=$remaining
  [ "$request_seconds" -le 30 ] || request_seconds=30
  command_seconds=$request_seconds
  local args=()
  for arg in "$@"; do
    case "$arg" in
      --request-timeout=*|--timeout=*|--wait=*) ;;
      *) args+=("$arg") ;;
    esac
    [ "$arg" != delete ] || is_delete=true
    [ "$arg" != rollout ] || is_rollout=true
  done
  # Acceptance is followed by absence checks below; finalizers must not spend
  # the restoration budget. This override applies only inside lifecycle cleanup.
  if [ "$is_delete" = true ]; then args+=(--wait=false); fi
  if [ "$is_rollout" = true ]; then
    command_seconds=$remaining
    request_seconds=$remaining
    args+=(--timeout="${remaining}s")
  fi
  node "$HCC_CLEANUP_COMMAND_RUNNER" "$command_seconds" "$KUBECTL_BIN" \
    --context "$E2E_KUBECONTEXT" "${args[@]}" \
    --request-timeout="${request_seconds}s"
}

hcc_cleanup_wait_until() {
  local requested=$1 description=$2 deadline remaining
  shift 2
  deadline=$((SECONDS + requested))
  [ "$deadline" -le "$HCC_CLEANUP_PHASE_DEADLINE" ] || deadline=$HCC_CLEANUP_PHASE_DEADLINE
  while [ "$SECONDS" -lt "$deadline" ]; do
    "$@" && return 0
    remaining=$((deadline - SECONDS))
    [ "$remaining" -gt 0 ] || break
    sleep 1
  done
  printf 'HCC cleanup deadline reached: %s\n' "$description" >&2
  return 1
}

hcc_lifecycle_fixture_absent() {
  local remaining host
  if [ "$NP604_CREATED" = 1 ]; then
    np604_resources_absent || return 1
    remaining="$(kctl get mcpserver,secret -n "$MCP_NS" \
      -l "e2e.clerum.io/suite=hcc-np604,e2e.clerum.io/run=${RUN_ID}" -o name)" || return 1
    [ -z "$remaining" ] || return 1
  fi
  remaining="$(kctl get mcpserver,context,host -A \
    -l "e2e.clerum.io/suite=hcc-watch-churn,e2e.clerum.io/run=${RUN_ID}" -o name)" || return 1
  [ -z "$remaining" ] || return 1
  while IFS= read -r host; do
    [ -n "$host" ] || continue
    remaining="$(kctl get pod,deployment,service,serviceaccount,role,rolebinding,secret,pvc,networkpolicy \
      -A -l "clerum.io/host=${host}" -o name)" || return 1
    [ -z "$remaining" ] || return 1
  done <<<"$HCC_CLEANUP_HOSTS"
  if [ "$FLEET_CREATED" = 1 ]; then
    remaining="$(kctl get secret "$FLEET_SECRET" -n "$HOST_NS" --ignore-not-found -o name)" || return 1
    [ -z "$remaining" ] || return 1
  fi
  if [ "$PROXY_CREATED" = 1 ]; then
    remaining="$(kctl get deployment,service "$PROXY_NAME" -n "$HCC_NS" --ignore-not-found -o name)" || return 1
    [ -z "$remaining" ] || return 1
    remaining="$(kctl get pod -n "$HCC_NS" -l "app=${PROXY_NAME}" -o name)" || return 1
    [ -z "$remaining" ] || return 1
  fi
  if [ "${HCC_PR_A_TLS_CREATED:-0}" = 1 ]; then
    remaining="$(kctl get secret,configmap -n "$HCC_NS" \
      -l "e2e.clerum.io/suite=hcc-watch-churn,e2e.clerum.io/run=${RUN_ID}" -o name)" || return 1
    [ -z "$remaining" ] || return 1
  fi
  if [ "$PROBE_CREATED" = 1 ]; then
    remaining="$(kctl get pod "$PROBE_NAME" -n "$HCC_NS" --ignore-not-found -o name)" || return 1
    [ -z "$remaining" ] || return 1
  fi
  if [ "$PROXY_CREATED" = 1 ] || [ "$PROBE_CREATED" = 1 ]; then
    remaining="$(kctl get networkpolicy "$PROXY_EGRESS_NP" "$HCC_PROXY_NP" "$PROBE_EGRESS_NP" \
      -n "$HCC_NS" --ignore-not-found -o name)" || return 1
    [ -z "$remaining" ] || return 1
  fi
  return 0
}

# Fixed budget: restore150 + fixtures100 + finalization20 = 270 seconds.
# The lifecycle target reserves 300 seconds of runner grace. Restoring HCC
# always precedes owner-driven fixture removal, which may never converge.
cleanup_hcc_lifecycle() (
  local status=$1 cleanup_failed=0 restore_ok=1 started=$SECONDS
  local HCC_CLEANUP_PHASE_DEADLINE=$((SECONDS + 150)) HCC_CLEANUP_HOSTS=''
  trap - EXIT
  trap ':' TERM INT HUP QUIT
  set +e
  kctl() { hcc_cleanup_kctl "$@"; }
  wait_until() { hcc_cleanup_wait_until "$@"; }

  # These are this shell's own capture children. Do not wait on a stalled stream.
  [ -z "${HCC_LOG_STREAM_PID:-}" ] || kill "$HCC_LOG_STREAM_PID" 2>/dev/null
  [ -z "${NP604_WATCH_PID:-}" ] || kill "$NP604_WATCH_PID" 2>/dev/null
  NP604_WATCH_PID=''
  if [ "${E2E_HCC_PR_A:-0}" = 1 ] &&
    { [ "$HCC_PATCHED" = 1 ] || [ "$HCC_SCALED_DOWN" = 1 ]; } && ! hcc_pr_a_identity_current; then
    restore_ok=0
  fi
  if [ "$HCC_PATCHED" = 1 ] && { [ "${E2E_HCC_PR_A:-0}" != 1 ] || [ "$restore_ok" = 1 ]; }; then
    restore_hcc_after_churn || restore_ok=0
  fi
  if { [ "$HCC_SCALED_DOWN" = 1 ] || [ "$HCC_PATCHED" = 1 ]; } &&
    { [ "${E2E_HCC_PR_A:-0}" != 1 ] || [ "$restore_ok" = 1 ]; }; then
    kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" \
      --replicas="${ORIGINAL_REPLICAS:-1}" >/dev/null 2>&1 || restore_ok=0
    kctl rollout status deployment "$HCC_DEPLOY" -n "$HCC_NS" \
      --timeout=150s >/dev/null 2>&1 || restore_ok=0
  fi
  if [ "$restore_ok" != 1 ]; then
    print_repair_instructions
    cleanup_failed=1
  fi

  # Preserve the proxy if HCC might still depend on its redirected API route.
  if [ "$restore_ok" = 1 ]; then
    HCC_CLEANUP_PHASE_DEADLINE=$((started + 250))
    if [ "$FLEET_CREATED" = 1 ]; then
      HCC_CLEANUP_HOSTS="$(kctl get host -A \
        -l "e2e.clerum.io/suite=hcc-watch-churn,e2e.clerum.io/run=${RUN_ID}" \
        -o 'jsonpath={range .items[*]}{.metadata.name}{"\n"}{end}')" || cleanup_failed=1
    fi
    np604_cleanup || cleanup_failed=1
    [ "$FLEET_CREATED" != 1 ] || delete_synthetic_fleet || cleanup_failed=1
    [ "$PROXY_CREATED" != 1 ] || kctl delete deployment,service "$PROXY_NAME" -n "$HCC_NS" --ignore-not-found >/dev/null 2>&1 || cleanup_failed=1
    [ "$PROBE_CREATED" != 1 ] || kctl delete pod "$PROBE_NAME" -n "$HCC_NS" --ignore-not-found >/dev/null 2>&1 || cleanup_failed=1
    if [ "$PROXY_CREATED" = 1 ] || [ "$PROBE_CREATED" = 1 ]; then
      kctl delete networkpolicy "$PROXY_EGRESS_NP" "$HCC_PROXY_NP" "$PROBE_EGRESS_NP" \
        -n "$HCC_NS" --ignore-not-found >/dev/null 2>&1 || cleanup_failed=1
    fi
    if [ "${HCC_PR_A_TLS_CREATED:-0}" = 1 ]; then
      kctl delete secret,configmap -n "$HCC_NS" \
        -l "e2e.clerum.io/suite=hcc-watch-churn,e2e.clerum.io/run=${RUN_ID}" \
        --ignore-not-found >/dev/null 2>&1 || cleanup_failed=1
    fi
    wait_until 100 'lifecycle fixture resources removed' hcc_lifecycle_fixture_absent || cleanup_failed=1
  fi

  HCC_CLEANUP_PHASE_DEADLINE=$((started + 270))
  if { [ "$status" != 0 ] || [ "$cleanup_failed" != 0 ]; } &&
    [ "${E2E_HCC_PR_A:-0}" = 1 ] && [ "${NP604_CREATED:-0}" = 1 ] &&
    [ -d "${NP604_EVIDENCE:-}" ] && [ -r "$HCC_LOG_BUFFER" ]; then
    # Keep structured evidence for later phases too, before deleting the raw
    # buffer. Diagnostics do not replace or change the existing failure verdict.
    hcc_pr_a_recovery_checkpoint 0 "$NP604_EVIDENCE/failure-recovery-observation.jsonl" ||
      printf 'PR_A_FAILURE_CHECKPOINT=UNAVAILABLE\n' >&2
  fi
  finalize_hcc_watch_gate_lock "$cleanup_failed" "$restore_ok" || cleanup_failed=1
  print_results || cleanup_failed=1
  if [ "$status" -eq 0 ] && [ "$cleanup_failed" != 0 ]; then status=1; fi
  if [ "$cleanup_failed" = 0 ] && [ -n "${HCC_PR_A_CONFIG_SNAPSHOT:-}" ]; then
    rm -f "$HCC_PR_A_CONFIG_SNAPSHOT"
  fi
  rm -f "$HCC_LOG_BUFFER" "$READY_SERIES"
  return "$status"
)