Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

34.1 KB
Raw Blame
#!/usr/bin/env bash

# Run Pulse hot-dev in a managed background session (macOS/Linux).
#
# Usage:
#   npm run dev                             # Canonical managed dev runtime
#   npm run dev:lab                         # LAN-bound lab-agent runtime
#   npm run dev:status
#   npm run dev:logs
#   npm run dev:restart
#   npm run dev:restart:lab
#   npm run dev:backend-restart
#   npm run dev:verify
#   npm run dev:verify:lab
#   ./scripts/hot-dev-bg.sh <command>       # Direct troubleshooting only

set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ROOT_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"

# shellcheck disable=SC1091
source "${SCRIPT_DIR}/lib/hot-dev-runtime.sh"

DEFAULT_HOT_DEV_BG_PID_FILE="${ROOT_DIR}/tmp/hot-dev.bg.pid"
PID_FILE="${HOT_DEV_BG_PID_FILE:-${DEFAULT_HOT_DEV_BG_PID_FILE}}"
LOG_FILE="${HOT_DEV_BG_LOG_FILE:-${ROOT_DIR}/tmp/hot-dev.bg.log}"
HOT_DEV_VERIFY_LOCK="${HOT_DEV_VERIFY_LOCK_FILE:-${ROOT_DIR}/tmp/hot-dev.verify.lock}"
HOT_DEV_LAB_MODE_FILE="${HOT_DEV_LAB_MODE_FILE:-${ROOT_DIR}/tmp/hot-dev.lab-agents-mode}"

hot_dev_configure_network_defaults

MACOS_HOT_DEV_LABEL="com.pulse.hot-dev"

log() {
  printf "[hot-dev-bg] %s\n" "$*"
}

fail() {
  printf "[hot-dev-bg] ERROR: %s\n" "$*" >&2
  exit 1
}

http_status_code() {
  local url="$1"
  local output code
  output="$(curl -sS -m 2 -o /dev/null -w $'\n%{http_code}' "${url}" 2>&1 || true)"
  code="${output##*$'\n'}"
  if [[ ! "${code}" =~ ^[0-9][0-9][0-9]$ ]]; then
    code="000"
  fi
  if [[ "${code}" == "000" && "${output}" == *"Operation not permitted"* ]]; then
    code="blocked"
  fi
  printf "%s\n" "${code}"
}

http_status_ok() {
  local code="$1"
  [[ "${code}" =~ ^2[0-9][0-9]$ || "${code}" =~ ^3[0-9][0-9]$ ]]
}

http_status_blocked() {
  local code="$1"
  [[ "${code}" == "blocked" ]]
}

is_port_listening() {
  local port="$1"
  lsof -nP -iTCP:"${port}" -sTCP:LISTEN >/dev/null 2>&1
}

listener_pids() {
  local port="$1"
  lsof -nP -t -iTCP:"${port}" -sTCP:LISTEN 2>/dev/null | sort -u
}

process_group_id() {
  local pid="$1"
  ps -o pgid= -p "${pid}" 2>/dev/null | tr -d '[:space:]'
}

process_command() {
  local pid="$1"
  ps -o command= -p "${pid}" 2>/dev/null | sed 's/^[[:space:]]*//'
}

process_parent_id() {
  local pid="$1"
  ps -o ppid= -p "${pid}" 2>/dev/null | tr -d '[:space:]'
}

process_probe_denied() {
  local output="$1"
  [[ "${output}" == *"Operation not permitted"* || "${output}" == *"operation not permitted"* || "${output}" == *"not permitted"* ]]
}

pid_signal_probe() {
  local pid="${1:-}"
  local output

  [[ -n "${pid}" ]] || return 1
  if output="$(kill -0 "${pid}" 2>&1)"; then
    return 0
  fi
  if process_probe_denied "${output}"; then
    return 2
  fi
  return 1
}

pid_may_be_running() {
  local status
  pid_signal_probe "${1:-}"
  status=$?
  [[ "${status}" -eq 0 || "${status}" -eq 2 ]]
}

process_inspection_restricted() {
  local output

  if output="$(ps -o pid= -p "$$" 2>&1 >/dev/null)"; then
    return 1
  fi
  process_probe_denied "${output}"
}

managed_supervisor_pids() {
  local pid command

  while IFS= read -r pid; do
    [[ -n "${pid}" ]] || continue
    [[ "${pid}" != "$$" ]] || continue
    command="$(process_command "${pid}")"
    case "${command}" in
      "bash ${ROOT_DIR}/scripts/hot-dev-bg.sh supervise"|"/bin/bash ${ROOT_DIR}/scripts/hot-dev-bg.sh supervise"|"${ROOT_DIR}/scripts/hot-dev-bg.sh supervise")
        printf "%s\n" "${pid}"
        ;;
    esac
  done < <(pgrep -f "${ROOT_DIR}/scripts/hot-dev-bg.sh supervise" 2>/dev/null | sort -n)
}

discover_managed_supervisor_pid() {
  local pid

  while IFS= read -r pid; do
    [[ -n "${pid}" ]] || continue
    printf "%s\n" "${pid}"
    return 0
  done < <(managed_supervisor_pids)

  return 1
}

recover_managed_pid_file() {
  local discovered_pid

  [[ "${PID_FILE}" == "${DEFAULT_HOT_DEV_BG_PID_FILE}" ]] || return 1
  discovered_pid="$(discover_managed_supervisor_pid || true)"
  [[ -n "${discovered_pid}" ]] || return 1
  pid_may_be_running "${discovered_pid}" || return 1
  mkdir -p "$(dirname "${PID_FILE}")"
  printf "%s\n" "${discovered_pid}" > "${PID_FILE}"
  return 0
}

array_contains() {
  local needle="${1:-}"
  shift || true

  local item
  for item in "$@"; do
    [[ "${item}" == "${needle}" ]] && return 0
  done

  return 1
}

safe_kill_target() {
  local signal="${1:-TERM}"
  local target_pgid="${2:-}"
  local target_pid="${3:-}"
  local current_pgid=""

  if [[ -n "${target_pgid}" ]]; then
    current_pgid="$(process_group_id "$$")"
    if [[ -n "${current_pgid}" && "${target_pgid}" != "${current_pgid}" ]]; then
      kill "-${signal}" "-${target_pgid}" 2>/dev/null && return 0
    fi
  fi

  [[ -n "${target_pid}" ]] || return 1
  kill "-${signal}" "${target_pid}" 2>/dev/null
}

current_shell_has_ancestor() {
  local target_pid="${1:-}"
  local current_pid="$$"
  local parent_pid

  [[ -n "${target_pid}" ]] || return 1

  while [[ -n "${current_pid}" && "${current_pid}" != "1" ]]; do
    parent_pid="$(process_parent_id "${current_pid}")"
    [[ -n "${parent_pid}" && "${parent_pid}" != "${current_pid}" ]] || break
    if [[ "${parent_pid}" == "${target_pid}" ]]; then
      return 0
    fi
    current_pid="${parent_pid}"
  done

  return 1
}

takeover_should_signal_listeners_only() {
  local target_pgid="${1:-}"
  local target_pid="${2:-}"
  local current_pgid=""

  current_pgid="$(process_group_id "$$")"
  if [[ -n "${target_pgid}" && -n "${current_pgid}" && "${target_pgid}" == "${current_pgid}" ]]; then
    return 0
  fi

  current_shell_has_ancestor "${target_pid}"
}

pid_is_managed() {
  local pid="$1"
  local session_pid="$2"
  local current_pid="${pid}"
  local parent_pid

  [[ -n "${session_pid}" ]] || return 1

  while [[ -n "${current_pid}" && "${current_pid}" != "1" ]]; do
    if [[ "${current_pid}" == "${session_pid}" ]]; then
      return 0
    fi
    if [[ "$(process_group_id "${current_pid}")" == "${session_pid}" ]]; then
      return 0
    fi
    parent_pid="$(process_parent_id "${current_pid}")"
    [[ -n "${parent_pid}" && "${parent_pid}" != "${current_pid}" ]] || break
    current_pid="${parent_pid}"
  done

  return 1
}

port_has_managed_listener() {
  local port="$1"
  local session_pid="$2"
  local listener_pid

  while IFS= read -r listener_pid; do
    [[ -n "${listener_pid}" ]] || continue
    if pid_is_managed "${listener_pid}" "${session_pid}"; then
      return 0
    fi
  done < <(listener_pids "${port}")

  return 1
}

port_has_unmanaged_listener() {
  local port="$1"
  local session_pid="$2"
  local listener_pid

  if [[ -n "${session_pid}" ]] && process_inspection_restricted; then
    return 1
  fi

  while IFS= read -r listener_pid; do
    [[ -n "${listener_pid}" ]] || continue
    if ! pid_is_managed "${listener_pid}" "${session_pid}"; then
      return 0
    fi
  done < <(listener_pids "${port}")

  return 1
}

first_managed_listener_pid() {
  local port="$1"
  local session_pid="$2"
  local listener_pid

  while IFS= read -r listener_pid; do
    [[ -n "${listener_pid}" ]] || continue
    if pid_is_managed "${listener_pid}" "${session_pid}"; then
      printf "%s\n" "${listener_pid}"
      return 0
    fi
  done < <(listener_pids "${port}")

  return 1
}

wait_for_managed_listener() {
  local port="$1"
  local session_pid="$2"
  local timeout_seconds="${3:-60}"
  local checks=$((timeout_seconds * 2))

  while (( checks > 0 )); do
    if port_has_managed_listener "${port}" "${session_pid}"; then
      return 0
    fi
    sleep 0.5
    checks=$((checks - 1))
  done

  return 1
}

adopt_managed_runtime_with_listener() {
  local port="$1"
  local candidate_pid

  [[ "${PID_FILE}" == "${DEFAULT_HOT_DEV_BG_PID_FILE}" ]] || return 1

  while IFS= read -r candidate_pid; do
    [[ -n "${candidate_pid}" ]] || continue
    pid_may_be_running "${candidate_pid}" || continue
    if port_has_managed_listener "${port}" "${candidate_pid}"; then
      mkdir -p "$(dirname "${PID_FILE}")"
      printf "%s\n" "${candidate_pid}" > "${PID_FILE}"
      log "Adopted managed runtime supervisor after handoff (pid: ${candidate_pid})"
      return 0
    fi
  done < <(managed_supervisor_pids)

  return 1
}

wait_for_managed_runtime_listener() {
  local port="$1"
  local session_pid="$2"
  local timeout_seconds="${3:-60}"
  local checks=$((timeout_seconds * 2))

  while (( checks > 0 )); do
    if [[ -n "${session_pid}" ]] && port_has_managed_listener "${port}" "${session_pid}"; then
      return 0
    fi
    if adopt_managed_runtime_with_listener "${port}"; then
      return 0
    fi
    sleep 0.5
    checks=$((checks - 1))
  done

  return 1
}

is_running() {
  if [[ ! -f "${PID_FILE}" ]]; then
    recover_managed_pid_file
    return $?
  fi
  local pid
  pid="$(cat "${PID_FILE}" 2>/dev/null || true)"
  if [[ -n "${pid}" ]] && pid_may_be_running "${pid}"; then
    return 0
  fi
  rm -f "${PID_FILE}"
  recover_managed_pid_file
}

managed_session_pid() {
  if is_running; then
    cat "${PID_FILE}" 2>/dev/null || true
  fi
}

describe_listener() {
  local port="$1"
  local session_pid="${2:-}"
  local found=0
  local listener_pid pgid owner command

  while IFS= read -r listener_pid; do
    [[ -n "${listener_pid}" ]] || continue
    found=1
    pgid="$(process_group_id "${listener_pid}")"
    owner="unmanaged"
    if pid_is_managed "${listener_pid}" "${session_pid}"; then
      owner="managed"
    elif [[ -n "${session_pid}" ]] && process_inspection_restricted; then
      owner="unknown"
    fi
    command="$(process_command "${listener_pid}")"
    log "Port ${port}: ${owner} listener pid=${listener_pid} pgid=${pgid:-unknown} cmd=${command:-unknown}"
  done < <(listener_pids "${port}")

  if [[ "${found}" -eq 1 ]]; then
    return 0
  fi
  return 1
}

has_any_listener() {
  local port="$1"
  local listener_pid

  while IFS= read -r listener_pid; do
    [[ -n "${listener_pid}" ]] || continue
    return 0
  done < <(listener_pids "${port}")

  return 1
}

has_unmanaged_listeners() {
  local session_pid="${1:-}"
  local port

  for port in "${FRONTEND_DEV_PORT}" "${PULSE_DEV_API_PORT}"; do
    if has_any_listener "${port}" && port_has_unmanaged_listener "${port}" "${session_pid}"; then
      return 0
    fi
  done

  return 1
}

find_hot_dev_ancestor() {
  local pid="$1"
  local current_pid="${pid}"
  local command parent_pid

  while [[ -n "${current_pid}" && "${current_pid}" != "1" ]]; do
    command="$(process_command "${current_pid}")"
    if [[ "${command}" == *"/scripts/hot-dev.sh"* || "${command}" == "bash scripts/hot-dev.sh" ]]; then
      printf "%s\n" "${current_pid}"
      return 0
    fi
    parent_pid="$(process_parent_id "${current_pid}")"
    [[ -n "${parent_pid}" && "${parent_pid}" != "${current_pid}" ]] || break
    current_pid="${parent_pid}"
  done

  return 1
}

hot_dev_root_pids() {
  pgrep -f '(^|/)(hot-dev\.sh|bash scripts/hot-dev\.sh)$|/scripts/hot-dev\.sh' 2>/dev/null | sort -u
}

stop_hot_dev_sessions() {
  local signal="${1:-TERM}"
  local root_pid root_pgid root_command target_key
  local seen=()

  while IFS= read -r root_pid; do
    [[ -n "${root_pid}" ]] || continue
    root_pgid="$(process_group_id "${root_pid}")"
    if [[ -n "${root_pgid}" ]]; then
      if takeover_should_signal_listeners_only "${root_pgid}" "${root_pid}"; then
        continue
      fi
      target_key="pgid:${root_pgid}"
      if array_contains "${target_key}" "${seen[@]-}"; then
        continue
      fi
      seen+=("${target_key}")
      root_command="$(process_command "${root_pid}")"
      log "Takeover sending ${signal} to hot-dev session group ${root_pgid} rooted at pid=${root_pid} cmd=${root_command:-unknown}"
      safe_kill_target "${signal}" "${root_pgid}" "${root_pid}" || true
      continue
    fi

    target_key="pid:${root_pid}"
    if array_contains "${target_key}" "${seen[@]-}"; then
      continue
    fi
    seen+=("${target_key}")
    root_command="$(process_command "${root_pid}")"
    log "Takeover sending ${signal} to hot-dev root pid=${root_pid} cmd=${root_command:-unknown}"
    kill "-${signal}" "${root_pid}" 2>/dev/null || true
  done < <(hot_dev_root_pids)
}

launchd_hot_dev_target() {
  printf "gui/%s/%s\n" "$(id -u)" "${MACOS_HOT_DEV_LABEL}"
}

launchd_hot_dev_active() {
  [[ "$(uname -s)" == "Darwin" ]] || return 1
  launchctl print "$(launchd_hot_dev_target)" >/dev/null 2>&1
}

stop_launchd_hot_dev_job() {
  launchd_hot_dev_active || return 0
  local target
  target="$(launchd_hot_dev_target)"
  log "Takeover booting out launchd job ${MACOS_HOT_DEV_LABEL}"
  launchctl bootout "${target}" >/dev/null 2>&1 || launchctl remove "${MACOS_HOT_DEV_LABEL}" >/dev/null 2>&1 || true
}

stop_takeover_targets() {
  local signal="${1:-TERM}"
  local port listener_pid ancestor_pid target_key target_pid target_pgid target_command
  local seen=()

  for port in "${FRONTEND_DEV_PORT}" "${PULSE_DEV_API_PORT}"; do
    while IFS= read -r listener_pid; do
      [[ -n "${listener_pid}" ]] || continue

      ancestor_pid="$(find_hot_dev_ancestor "${listener_pid}" || true)"
      if [[ -n "${ancestor_pid}" ]]; then
        target_pgid="$(process_group_id "${ancestor_pid}")"
        [[ -n "${target_pgid}" ]] || continue
        if ! takeover_should_signal_listeners_only "${target_pgid}" "${ancestor_pid}"; then
          target_key="pgid:${target_pgid}"
          if array_contains "${target_key}" "${seen[@]-}"; then
            continue
          fi
          seen+=("${target_key}")
          target_command="$(process_command "${ancestor_pid}")"
          log "Takeover sending ${signal} to hot-dev process group ${target_pgid} rooted at pid=${ancestor_pid} cmd=${target_command:-unknown}"
          safe_kill_target "${signal}" "${target_pgid}" "${ancestor_pid}" || true
          continue
        fi
      fi

      target_key="pid:${listener_pid}"
      if array_contains "${target_key}" "${seen[@]-}"; then
        continue
      fi
      seen+=("${target_key}")
      target_command="$(process_command "${listener_pid}")"
      log "Takeover sending ${signal} to listener pid=${listener_pid} cmd=${target_command:-unknown}"
      kill "-${signal}" "${listener_pid}" 2>/dev/null || true
    done < <(listener_pids "${port}")
  done
}

wait_for_ports_to_clear() {
  local timeout_seconds="${1:-10}"
  local checks=$((timeout_seconds * 2))
  local port

  while (( checks > 0 )); do
    local any_listeners="false"
    for port in "${FRONTEND_DEV_PORT}" "${PULSE_DEV_API_PORT}"; do
      if has_any_listener "${port}"; then
        any_listeners="true"
        break
      fi
    done
    if [[ "${any_listeners}" == "false" ]]; then
      return 0
    fi
    sleep 0.5
    checks=$((checks - 1))
  done

  return 1
}

require_python() {
  command -v python3 >/dev/null 2>&1 || fail "python3 is required"
}

start_hot_dev_child() {
  FRONTEND_DEV_HOST="${FRONTEND_DEV_HOST}" \
  FRONTEND_DEV_PORT="${FRONTEND_DEV_PORT}" \
  PULSE_DEV_API_HOST="${PULSE_DEV_API_HOST}" \
  PULSE_DEV_API_PORT="${PULSE_DEV_API_PORT}" \
  PULSE_DEV_API_URL="${PULSE_DEV_API_URL}" \
  PULSE_DEV_WS_URL="${PULSE_DEV_WS_URL}" \
  PULSE_DEV_LAN="${PULSE_DEV_LAN:-}" \
  PULSE_DEV_LAB_AGENTS="${PULSE_DEV_LAB_AGENTS:-}" \
  BIND_ADDRESS="${BIND_ADDRESS:-}" \
  ALLOWED_ORIGINS="${ALLOWED_ORIGINS:-}" \
  LOG_LEVEL="${LOG_LEVEL:-}" \
  PULSE_ENABLE_PROXMOX_GUEST_DOCKER_DETECTION="${PULSE_ENABLE_PROXMOX_GUEST_DOCKER_DETECTION:-}" \
  PULSE_ENABLE_PROXMOX_GUEST_DOCKER_INVENTORY="${PULSE_ENABLE_PROXMOX_GUEST_DOCKER_INVENTORY:-}" \
  PULSE_PROXMOX_GUEST_DOCKER_INVENTORY_VMIDS="${PULSE_PROXMOX_GUEST_DOCKER_INVENTORY_VMIDS:-}" \
  HOT_DEV_LAB_MODE_FILE="${HOT_DEV_LAB_MODE_FILE:-}" \
  HOT_DEV_SKIP_NPM_CLEANUP=true \
  ROOT_DIR="${ROOT_DIR}" \
  python3 - <<'PY' &
import os

os.setsid()
os.execve(
    os.path.join(os.environ["ROOT_DIR"], "scripts", "hot-dev.sh"),
    [os.path.join(os.environ["ROOT_DIR"], "scripts", "hot-dev.sh")],
    os.environ.copy(),
)
PY
  HOT_DEV_CHILD_PID="$!"
}

stop_hot_dev_child() {
  local child_pid="${1:-}"
  local retries=30

  [[ -n "${child_pid}" ]] || return 0
  kill -TERM "-${child_pid}" 2>/dev/null || kill -TERM "${child_pid}" 2>/dev/null || true

  while (( retries > 0 )); do
    if ! kill -0 "-${child_pid}" 2>/dev/null && ! kill -0 "${child_pid}" 2>/dev/null; then
      return 0
    fi
    sleep 0.5
    retries=$((retries - 1))
  done

  kill -KILL "-${child_pid}" 2>/dev/null || kill -KILL "${child_pid}" 2>/dev/null || true
}

run_supervised_session() {
  local restart_delay="${HOT_DEV_BG_SUPERVISOR_RESTART_DELAY:-1}"
  local child_pid=""
  local child_exit=0
  local shutdown_requested="false"

  supervisor_stop() {
    shutdown_requested="true"
    if [[ -n "${child_pid}" ]] && kill -0 "${child_pid}" 2>/dev/null; then
      stop_hot_dev_child "${child_pid}"
      wait "${child_pid}" 2>/dev/null || true
    fi
    exit 0
  }

  trap supervisor_stop TERM INT HUP

  while true; do
    HOT_DEV_CHILD_PID=""
    start_hot_dev_child
    child_pid="${HOT_DEV_CHILD_PID:-}"
    [[ -n "${child_pid}" ]] || fail "Managed runtime supervisor failed to start hot-dev"
    log "Supervisor started hot-dev child (pid: ${child_pid})"

    if wait "${child_pid}"; then
      child_exit=0
    else
      child_exit=$?
    fi

    if [[ "${shutdown_requested}" == "true" ]]; then
      exit 0
    fi

    stop_hot_dev_child "${child_pid}"
    log "Managed runtime child exited unexpectedly (pid: ${child_pid}, exit: ${child_exit}). Restarting..."
    sleep "${restart_delay}"
  done
}

spawn_detached_supervisor() {
  FRONTEND_DEV_HOST="${FRONTEND_DEV_HOST}" \
  FRONTEND_DEV_PORT="${FRONTEND_DEV_PORT}" \
  PULSE_DEV_API_HOST="${PULSE_DEV_API_HOST}" \
  PULSE_DEV_API_PORT="${PULSE_DEV_API_PORT}" \
  PULSE_DEV_API_URL="${PULSE_DEV_API_URL}" \
  PULSE_DEV_WS_URL="${PULSE_DEV_WS_URL}" \
  PULSE_DEV_LAN="${PULSE_DEV_LAN:-}" \
  PULSE_DEV_LAB_AGENTS="${PULSE_DEV_LAB_AGENTS:-}" \
  BIND_ADDRESS="${BIND_ADDRESS:-}" \
  ALLOWED_ORIGINS="${ALLOWED_ORIGINS:-}" \
  LOG_LEVEL="${LOG_LEVEL:-}" \
  PULSE_ENABLE_PROXMOX_GUEST_DOCKER_DETECTION="${PULSE_ENABLE_PROXMOX_GUEST_DOCKER_DETECTION:-}" \
  PULSE_ENABLE_PROXMOX_GUEST_DOCKER_INVENTORY="${PULSE_ENABLE_PROXMOX_GUEST_DOCKER_INVENTORY:-}" \
  PULSE_PROXMOX_GUEST_DOCKER_INVENTORY_VMIDS="${PULSE_PROXMOX_GUEST_DOCKER_INVENTORY_VMIDS:-}" \
  HOT_DEV_LAB_MODE_FILE="${HOT_DEV_LAB_MODE_FILE:-}" \
  ROOT_DIR="${ROOT_DIR}" \
  LOG_FILE="${LOG_FILE}" \
  HOT_DEV_BG_SCRIPT="${ROOT_DIR}/scripts/hot-dev-bg.sh" \
  python3 - <<'PY'
import os
import subprocess

root_dir = os.environ["ROOT_DIR"]
log_file = os.environ["LOG_FILE"]
cmd = [os.environ["HOT_DEV_BG_SCRIPT"], "supervise"]

env = os.environ.copy()

with open(log_file, "ab", buffering=0) as log:
    proc = subprocess.Popen(
        cmd,
        cwd=root_dir,
        stdin=subprocess.DEVNULL,
        stdout=log,
        stderr=subprocess.STDOUT,
        start_new_session=True,
        env=env,
    )

print(proc.pid)
PY
}

start_bg() {
  local takeover="${1:-false}"
  local browser_url lan_browser_url
  require_python
  mkdir -p "$(dirname "${PID_FILE}")"
  touch "${LOG_FILE}"

  if is_running; then
    log "Already running (pid: $(cat "${PID_FILE}"))"
    return 0
  fi

  # Clear stale PID file before fresh start.
  rm -f "${PID_FILE}"

  if has_unmanaged_listeners ""; then
    if [[ "${takeover}" != "true" ]]; then
      log "Unmanaged listeners are already using the dev ports."
      describe_listener "${FRONTEND_DEV_PORT}" "" || true
      describe_listener "${PULSE_DEV_API_PORT}" "" || true
      fail "Refusing to take over unmanaged listeners. Stop them first or rerun with: npm run dev"
    fi
    log "Taking over existing unmanaged listeners on the dev ports."
    describe_listener "${FRONTEND_DEV_PORT}" "" || true
    describe_listener "${PULSE_DEV_API_PORT}" "" || true
    stop_launchd_hot_dev_job
    stop_hot_dev_sessions TERM
    stop_takeover_targets TERM
    if ! wait_for_ports_to_clear 10; then
      log "Takeover escalation: dev ports are still occupied after TERM."
      stop_hot_dev_sessions KILL
      stop_takeover_targets KILL
      if ! wait_for_ports_to_clear 5; then
        log "Remaining listeners after takeover escalation:"
        describe_listener "${FRONTEND_DEV_PORT}" "" || true
        describe_listener "${PULSE_DEV_API_PORT}" "" || true
        fail "Unable to reclaim the dev ports for managed takeover"
      fi
    fi
  fi

  local pid
  pid="$(spawn_detached_supervisor)"

  [[ -n "${pid}" ]] || fail "Failed to start managed runtime supervisor"
  printf "%s\n" "${pid}" > "${PID_FILE}"

  log "Started managed runtime supervisor (pid: ${pid})"
  log "Waiting for managed backend listener on port ${PULSE_DEV_API_PORT}..."
  if ! wait_for_managed_runtime_listener "${PULSE_DEV_API_PORT}" "${pid}" 90; then
    log "Backend did not start. Showing recent logs:"
    tail -n 80 "${LOG_FILE}" || true
    fail "managed runtime startup failed"
  fi
  pid="$(managed_session_pid || true)"
  [[ -n "${pid}" ]] || pid="$(cat "${PID_FILE}" 2>/dev/null || true)"

  if ! wait_for_managed_runtime_listener "${FRONTEND_DEV_PORT}" "${pid}" 90; then
    log "Backend is up, but a managed frontend listener on port ${FRONTEND_DEV_PORT} is not ready yet."
    log "Check logs with: npm run dev:logs"
  fi

  browser_url="$(hot_dev_local_browser_url "${FRONTEND_DEV_HOST}" "${FRONTEND_DEV_PORT}")"
  lan_browser_url="$(hot_dev_lan_browser_url "${FRONTEND_DEV_HOST}" "${FRONTEND_DEV_PORT}" "${LAN_IP}")"
  log "Browser entrypoint: ${browser_url}"
  if [[ -n "${lan_browser_url}" ]]; then
    log "LAN browser entrypoint: ${lan_browser_url}"
  fi
  log "Managed backend:  http://127.0.0.1:${PULSE_DEV_API_PORT}"
  log "Logs: ${LOG_FILE}"
}

stop_bg() {
  if ! is_running; then
    rm -f "${PID_FILE}"
    log "Not running"
    return 0
  fi

  local pid
  pid="$(cat "${PID_FILE}")"

  # Kill the entire session/process group created by start_new_session=True.
  kill -TERM "-${pid}" 2>/dev/null || kill -TERM "${pid}" 2>/dev/null || true

  local retries=30
  while (( retries > 0 )); do
    if ! kill -0 "${pid}" 2>/dev/null; then
      break
    fi
    sleep 0.5
    retries=$((retries - 1))
  done

  if kill -0 "${pid}" 2>/dev/null; then
    kill -KILL "-${pid}" 2>/dev/null || kill -KILL "${pid}" 2>/dev/null || true
  fi

  rm -f "${PID_FILE}"
  log "Stopped hot-dev"
}

status_bg() {
  local managed_pid=""
  if is_running; then
    managed_pid="$(cat "${PID_FILE}")"
    log "Running (pid: ${managed_pid})"
  else
    log "Not running"
  fi

  if is_port_listening "${FRONTEND_DEV_PORT}"; then
    log "Frontend port ${FRONTEND_DEV_PORT} is listening"
  else
    log "Frontend port ${FRONTEND_DEV_PORT} is not listening"
  fi

  if is_port_listening "${PULSE_DEV_API_PORT}"; then
    log "Backend port ${PULSE_DEV_API_PORT} is listening"
  else
    log "Backend port ${PULSE_DEV_API_PORT} is not listening"
  fi

  local browser_url lan_browser_url frontend_url proxy_health_url backend_health_url
  browser_url="$(hot_dev_local_browser_url "${FRONTEND_DEV_HOST}" "${FRONTEND_DEV_PORT}")"
  lan_browser_url="$(hot_dev_lan_browser_url "${FRONTEND_DEV_HOST}" "${FRONTEND_DEV_PORT}" "${LAN_IP}")"
  frontend_url="${browser_url}/"
  proxy_health_url="${browser_url}/api/health"
  backend_health_url="http://127.0.0.1:${PULSE_DEV_API_PORT}/api/health"

  local frontend_code proxy_code backend_code local_probe_unavailable
  frontend_code="$(http_status_code "${frontend_url}")"
  proxy_code="$(http_status_code "${proxy_health_url}")"
  backend_code="$(http_status_code "${backend_health_url}")"
  local_probe_unavailable="false"
  if process_inspection_restricted && [[ "${frontend_code}" == "000" && "${backend_code}" == "000" ]] && is_port_listening "${FRONTEND_DEV_PORT}" && is_port_listening "${PULSE_DEV_API_PORT}"; then
    local_probe_unavailable="true"
    frontend_code="unavailable"
    proxy_code="unavailable"
    backend_code="unavailable"
  fi
  log "Browser entrypoint: ${browser_url}"
  if [[ -n "${lan_browser_url}" ]]; then
    log "LAN browser entrypoint: ${lan_browser_url}"
  fi
  log "Frontend shell HTTP: ${frontend_code}"
  log "Frontend proxy /api/health: ${proxy_code}"
  log "Backend /api/health: ${backend_code}"

  describe_listener "${FRONTEND_DEV_PORT}" "${managed_pid}" || true
  describe_listener "${PULSE_DEV_API_PORT}" "${managed_pid}" || true

  if launchd_hot_dev_active; then
    log "LaunchAgent ${MACOS_HOT_DEV_LABEL} is active"
  fi

  if [[ -z "${managed_pid}" ]] && has_unmanaged_listeners ""; then
    log "Detected unmanaged dev listeners. hot-dev-bg is not managing the current runtime."
  elif [[ -n "${managed_pid}" ]] && has_unmanaged_listeners "${managed_pid}"; then
    log "Detected split ownership: managed session is running, but one or more dev ports are owned by another process."
  fi

  if http_status_ok "${frontend_code}" && http_status_ok "${proxy_code}" && http_status_ok "${backend_code}"; then
    log "Runtime summary: frontend shell, proxy, and backend are healthy. Use ${browser_url} in the browser."
  elif [[ "${local_probe_unavailable}" == "true" ]]; then
    log "Runtime summary: local HTTP probes are unavailable from this restricted shell, but frontend and backend listeners are present. Use ${browser_url} in the browser or rerun status outside the restricted shell."
  elif http_status_blocked "${frontend_code}" || http_status_blocked "${proxy_code}" || http_status_blocked "${backend_code}"; then
    log "Runtime summary: local HTTP probes are blocked by this shell. Use ${browser_url} in the browser or rerun status outside the restricted shell."
  elif http_status_ok "${frontend_code}" && ! http_status_ok "${proxy_code}" && http_status_ok "${backend_code}"; then
    log "Runtime summary: frontend shell is up and the backend is healthy, but the frontend proxy path is unhealthy."
  elif http_status_ok "${frontend_code}" && ! http_status_ok "${backend_code}"; then
    log "Runtime summary: frontend shell is up, but the backend health endpoint is unavailable."
  elif ! http_status_ok "${frontend_code}" && http_status_ok "${backend_code}"; then
    log "Runtime summary: backend is healthy, but the frontend dev shell is unavailable. Use the backend URL only for direct API debugging."
  else
    log "Runtime summary: frontend shell and backend are not both healthy. Fix the runtime before trusting browser results."
  fi
}

logs_bg() {
  touch "${LOG_FILE}"
  tail -n 120 -f "${LOG_FILE}"
}

restart_backend_bg() {
  if ! is_running; then
    fail "Managed hot-dev session is not running"
  fi

  local session_pid backend_pid next_backend_pid restart_sentinel
  session_pid="$(managed_session_pid)"
  backend_pid="$(first_managed_listener_pid "${PULSE_DEV_API_PORT}" "${session_pid}" || true)"
  restart_sentinel="${ROOT_DIR}/tmp/hot-dev.restart"

  if [[ ! -f "${restart_sentinel}" ]]; then
    fail "Managed restart sentinel not found at ${restart_sentinel}"
  fi

  log "Requesting managed backend restart via hot-dev file watcher"
  touch "${restart_sentinel}"

  local checks=180
  while (( checks > 0 )); do
    next_backend_pid="$(first_managed_listener_pid "${PULSE_DEV_API_PORT}" "${session_pid}" || true)"
    if [[ -z "${backend_pid}" && -n "${next_backend_pid}" ]]; then
      break
    fi
    if [[ -n "${backend_pid}" && -n "${next_backend_pid}" && "${backend_pid}" != "${next_backend_pid}" ]]; then
      break
    fi
    sleep 0.5
    checks=$((checks - 1))
  done

  if [[ -n "${backend_pid}" && "${backend_pid}" == "${next_backend_pid:-}" ]]; then
    log "Managed backend restart did not replace the listener process. Showing recent logs:"
    tail -n 80 "${LOG_FILE}" || true
    fail "Managed backend restart did not replace the listener process on port ${PULSE_DEV_API_PORT}"
  fi

  log "Waiting for managed backend listener to become healthy on port ${PULSE_DEV_API_PORT}..."
  if ! wait_for_managed_listener "${PULSE_DEV_API_PORT}" "${session_pid}" 90; then
    log "Managed backend did not recover. Showing recent logs:"
    tail -n 80 "${LOG_FILE}" || true
    fail "Managed backend failed to recover after restart"
  fi

  local backend_health_url backend_code checks
  backend_health_url="http://127.0.0.1:${PULSE_DEV_API_PORT}/api/health"
  backend_code="000"
  checks=180
  while (( checks > 0 )); do
    backend_code="$(http_status_code "${backend_health_url}")"
    if http_status_ok "${backend_code}"; then
      break
    fi
    sleep 0.5
    checks=$((checks - 1))
  done

  if ! http_status_ok "${backend_code}"; then
    fail "Managed backend listener recovered but health probe returned ${backend_code}"
  fi

  log "Managed backend recovered and is healthy on port ${PULSE_DEV_API_PORT}"
}

runtime_healthy() {
  local frontend_code proxy_code backend_code managed_pid
  managed_pid="$(managed_session_pid)"

  [[ -n "${managed_pid}" ]] || return 1
  port_has_managed_listener "${FRONTEND_DEV_PORT}" "${managed_pid}" || return 1
  port_has_managed_listener "${PULSE_DEV_API_PORT}" "${managed_pid}" || return 1

  frontend_code="$(http_status_code "http://127.0.0.1:${FRONTEND_DEV_PORT}/")"
  proxy_code="$(http_status_code "http://127.0.0.1:${FRONTEND_DEV_PORT}/api/health")"
  backend_code="$(http_status_code "http://127.0.0.1:${PULSE_DEV_API_PORT}/api/health")"

  http_status_ok "${frontend_code}" && http_status_ok "${proxy_code}" && http_status_ok "${backend_code}"
}

write_verify_lock() {
  mkdir -p "$(dirname "${HOT_DEV_VERIFY_LOCK}")"
  cat > "${HOT_DEV_VERIFY_LOCK}" <<EOF
pid=$$
created_at=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
EOF
}

clear_verify_lock() {
  rm -f "${HOT_DEV_VERIFY_LOCK}"
}

run_verify_proof_command() {
  local verify_command integration_dir
  integration_dir="${ROOT_DIR}/tests/integration"
  verify_command="${HOT_DEV_BG_VERIFY_COMMAND:-}"

  if [[ -z "${verify_command}" ]]; then
    (
      cd "${integration_dir}"
      PULSE_E2E_USE_HOT_DEV=1 \
      PULSE_E2E_USERNAME="${PULSE_E2E_USERNAME:-admin}" \
      PULSE_E2E_PASSWORD="${PULSE_E2E_PASSWORD:-adminadminadmin}" \
      HOT_DEV_VERIFY_LOCK_FILE="${HOT_DEV_VERIFY_LOCK}" \
      node ./scripts/run-playwright.mjs \
        tests/16-dev-runtime-recovery.spec.ts \
        tests/17-proxmox-backups-layout.spec.ts \
        tests/18-patrol-runtime-state.spec.ts \
        --project=chromium
    )
    return
  fi

  (
    cd "${ROOT_DIR}"
    PULSE_E2E_USE_HOT_DEV=1 \
    PULSE_E2E_USERNAME="${PULSE_E2E_USERNAME:-admin}" \
    PULSE_E2E_PASSWORD="${PULSE_E2E_PASSWORD:-adminadminadmin}" \
    HOT_DEV_VERIFY_LOCK_FILE="${HOT_DEV_VERIFY_LOCK}" \
    bash -lc "${verify_command}"
  )
}

verify_bg() {
  local takeover="${1:-false}"
  local managed_pid verify_rc

  require_python

  if ! is_running; then
    log "Managed runtime is not running. Starting it before verification..."
    start_bg "${takeover}"
  else
    managed_pid="$(managed_session_pid)"
    if [[ -n "${managed_pid}" ]] && has_unmanaged_listeners "${managed_pid}"; then
      if [[ "${takeover}" != "true" ]]; then
        fail "Managed runtime has split ownership. Rerun with: npm run dev:verify"
      fi
      log "Managed runtime has split ownership. Reclaiming ports before verification..."
      stop_bg
      start_bg "${takeover}"
    elif ! runtime_healthy; then
      log "Managed runtime is unhealthy. Restarting it before verification..."
      stop_bg
      start_bg "${takeover}"
    fi
  fi

  if ! runtime_healthy; then
    status_bg
    fail "Managed runtime is not healthy enough to run browser verification"
  fi

  log "Running managed dev-runtime browser verification proofs..."
  write_verify_lock
  if run_verify_proof_command; then
    verify_rc=0
  else
    verify_rc=$?
  fi
  clear_verify_lock

  return "${verify_rc}"
}

launchd_session_bg() {
  local takeover="${1:-false}"
  local managed_pid poll_interval

  require_python

  stop_and_exit() {
    log "Launchd supervisor received termination signal; stopping managed runtime..."
    stop_bg
    exit 0
  }

  trap stop_and_exit TERM INT HUP

  if ! is_running; then
    log "Managed runtime is not running. Starting supervised session..."
    start_bg "${takeover}"
  else
    managed_pid="$(managed_session_pid)"
    if [[ -n "${managed_pid}" ]] && has_unmanaged_listeners "${managed_pid}"; then
      if [[ "${takeover}" != "true" ]]; then
        fail "Managed runtime has split ownership. Rerun with: npm run dev"
      fi
      log "Managed runtime has split ownership. Reclaiming ports before launchd supervision..."
      stop_bg
      start_bg "${takeover}"
    elif ! runtime_healthy; then
      log "Managed runtime is unhealthy. Restarting it before launchd supervision..."
      stop_bg
      start_bg "${takeover}"
    fi
  fi

  managed_pid="$(managed_session_pid)"
  [[ -n "${managed_pid}" ]] || fail "Managed runtime did not produce a session pid"

  poll_interval="${HOT_DEV_BG_SUPERVISOR_POLL_INTERVAL:-1}"
  log "Supervising managed runtime for launchd (pid: ${managed_pid})"

  while is_running; do
    sleep "${poll_interval}"
  done

  rm -f "${PID_FILE}"
  log "Managed runtime exited unexpectedly under launchd supervision"
  touch "${LOG_FILE}"
  tail -n 80 "${LOG_FILE}" || true
  return 1
}

usage() {
  cat <<EOF
Usage: $(basename "$0") <command>

Managed entrypoints:
  npm run dev
  npm run dev:lab
  npm run dev:status
  npm run dev:status:lab
  npm run dev:logs
  npm run dev:stop
  npm run dev:restart
  npm run dev:restart:lab
  npm run dev:backend-restart
  npm run dev:verify
  npm run dev:verify:lab
  npm run dev:foreground
  npm run dev:foreground:lab

Direct troubleshooting subcommands:
  start [--takeover]
  stop
  restart
  backend-restart
  launchd-session [--takeover]
  verify [--takeover]
  status
  logs
EOF
}

parse_takeover_flag() {
  local command="${1:-}"
  local flag="${2:-}"

  case "${command}" in
    start|restart|verify|launchd-session)
      if [[ -z "${flag}" ]]; then
        printf "false\n"
        return 0
      fi
      if [[ "${flag}" == "--takeover" ]]; then
        printf "true\n"
        return 0
      fi
      usage
      return 1
      ;;
    stop|backend-restart|status|logs|supervise)
      if [[ -n "${flag}" ]]; then
        usage
        return 1
      fi
      printf "false\n"
      return 0
      ;;
    *)
      printf "false\n"
      return 0
      ;;
  esac
}

main() {
  local command="${1:-}"
  local takeover
  takeover="$(parse_takeover_flag "$@")" || exit 1

  case "${command}" in
    supervise) run_supervised_session ;;
    start)
      start_bg "${takeover}"
      ;;
    stop) stop_bg ;;
    restart)
      if [[ "${takeover}" == "true" ]]; then
        stop_launchd_hot_dev_job
      fi
      stop_bg
      start_bg "${takeover}"
      ;;
    backend-restart) restart_backend_bg ;;
    launchd-session) launchd_session_bg "${takeover}" ;;
    verify) verify_bg "${takeover}" ;;
    status) status_bg ;;
    logs) logs_bg ;;
    *) usage; exit 1 ;;
  esac
}

if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
  main "$@"
fi