Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

39.1 KB
Raw Blame
#!/bin/bash
# hot-dev.sh - Foreground Pulse dev runtime escape hatch
#
# This script runs a local development environment with:
# - Go backend with auto-rebuild on file changes (via inotifywait)
# - Vite frontend dev server with HMR
# - Auto-detection of pulse-enterprise module for Pro features
# - Snapshot watcher (if scripts/watch-snapshot.sh exists)
#
# Environment Variables:
#   HOT_DEV_USE_PROD_DATA=true   Use /etc/pulse for data (sessions, config, etc.)
#   HOT_DEV_USE_PRO=true         Build Pro binary (default: true if module available)
#   HOT_DEV_ALLOW_OSS_FALLBACK=true  Allow fallback to OSS backend when Pro build fails
#   HOT_DEV_AUTH_USER=admin      Override the managed dev auth username
#   HOT_DEV_AUTH_PASS=...        Override the managed dev auth password or bcrypt hash
#   PULSE_MOCK_MODE=true         Render mock UI data while keeping real metrics history intact
#   PULSE_DATA_DIR=/path         Override data directory
#   PULSE_DEV_API_PORT=7655      Backend API port (default: 7655)
#   FRONTEND_DEV_PORT=5173       Frontend dev server port (default: 5173)
#   PULSE_DEV_LAN=true           Expose frontend/backend on the LAN for agent/mobile testing (default: false)
#   PULSE_DEV_LAB_AGENTS=true    Enable LAN binding and Proxmox LXC Docker inventory for installed lab agents
#   LOG_LEVEL=debug              Opt into verbose backend logs (default: info)
#   PULSE_DEV_DISABLE_BACKGROUND_AI=false
#                                   Allow automatic Patrol/discovery/alert AI in dev
#   HOT_DEV_BACKEND_HEALTH_STARTUP_GRACE_SECONDS=180
#                                   Backend /api/health grace after starts/restarts
#   HOT_DEV_BACKEND_UNHEALTHY_THRESHOLD=4
#                                   Consecutive failed /api/health probes before restart
#                                   (default tolerates ~20s of load-induced stall; test
#                                   suites and builds on the same machine routinely peg
#                                   the CPU and a kill mid-AI-stream bricks that chat turn)
#
# Pro Features Mode:
#   When pulse-enterprise repo exists and HOT_DEV_USE_PRO is not "false",
#   the script builds the Pro binary which includes:
#   - SQLite-based persistent audit logging
#   - RBAC (Role-Based Access Control)
#   - HMAC event signing for tamper detection
#
# Usage:
#   npm run dev                             # Canonical managed dev runtime
#   npm run dev:lab                         # LAN-bound lab-agent runtime
#   ./scripts/hot-dev.sh                    # Foreground/manual runtime troubleshooting
#   npm run dev:foreground:lab              # Foreground lab-agent troubleshooting
#   HOT_DEV_USE_PROD_DATA=true ./scripts/hot-dev.sh  # Foreground/manual runtime with production data
#
set -euo pipefail

SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)
ROOT_DIR=$(cd "${SCRIPT_DIR}/.." && pwd -P)
DEFAULT_PULSE_REPOS_DIR=$(cd "${ROOT_DIR}/.." && pwd -P)
SCRIPT_PATH="${SCRIPT_DIR}/$(basename "${BASH_SOURCE[0]}")"
SCRIPT_MTIME=$(stat -c %Y "${SCRIPT_PATH}" 2>/dev/null || stat -f %m "${SCRIPT_PATH}")

# shellcheck disable=SC1091
source "${SCRIPT_DIR}/lib/hot-dev-auth.sh"
# shellcheck disable=SC1091
source "${SCRIPT_DIR}/lib/hot-dev-runtime.sh"

HOT_DEV_LAB_MODE_FILE="${HOT_DEV_LAB_MODE_FILE:-${ROOT_DIR}/tmp/hot-dev.lab-agents-mode}"

# Keep build temp files off the boot volume. The Go linker's dsymutil step
# writes its DWARF output under TMPDIR (not GOTMPDIR), so on a machine with a
# space-constrained system disk the backend build fails with ENOSPC even
# though GOCACHE/GOTMPDIR already live on the big volume. Follow the
# machine's configured GOTMPDIR when one exists.
HOT_DEV_GO_TMPDIR="$(go env GOTMPDIR 2>/dev/null || true)"
if [[ -n "${HOT_DEV_GO_TMPDIR}" ]]; then
    mkdir -p "${HOT_DEV_GO_TMPDIR}" 2>/dev/null || true
    export TMPDIR="${HOT_DEV_GO_TMPDIR}"
fi

# --- Helper Functions ---

log_info() { printf "\033[0;34m[hot-dev]\033[0m %s\n" "$1"; }
log_warn() { printf "\033[0;33m[hot-dev] WARNING:\033[0m %s\n" "$1"; }
log_error() { printf "\033[0;31m[hot-dev] ERROR:\033[0m %s\n" "$1"; }

detect_encrypted_files() {
    local data_dir=$1
    local patterns=(nodes.enc* email.enc* webhooks.enc* oidc.enc* ai.enc*)
    local files=()

    for pattern in "${patterns[@]}"; do
        for path in "${data_dir}/${pattern}"; do
            if [[ -s "${path}" ]]; then
                files+=("$(basename "${path}")")
            fi
        done
    done

    printf '%s\n' "${files[@]}"
}

check_dependencies() {
    local missing=0
    for cmd in go npm lsof; do
        if ! command -v $cmd >/dev/null 2>&1; then
            log_error "$cmd is not installed but is required."
            missing=1
        fi
    done
    
    if [[ $missing -eq 1 ]]; then
        exit 1
    fi
}

build_backend_binary() {
    local build_output
    PULSE_REPOS_DIR="${PULSE_REPOS_DIR:-${DEFAULT_PULSE_REPOS_DIR}}"
    PRO_MODULE_DIR="${PULSE_REPOS_DIR}/pulse-enterprise"

    if [[ -d "${PRO_MODULE_DIR}" ]] && [[ ${HOT_DEV_USE_PRO:-true} == "true" ]]; then
        log_info "Building Pro binary (includes persistent audit logging)..."
        if build_output=$(cd "${PRO_MODULE_DIR}" && go build -buildvcs=false -o "${ROOT_DIR}/pulse" ./cmd/pulse-enterprise 2>&1); then
            printf "%s\n" "${build_output}" | grep -v "^#" || true
            PRO_BUILD_SUCCESS=true
            return 0
        fi

        printf "%s\n" "${build_output}" | grep -v "^#" || true
        if [[ ${HOT_DEV_ALLOW_OSS_FALLBACK:-false} != "true" ]]; then
            log_error "Pro build failed; refusing to fall back to the OSS backend."
            log_error "Fix pulse-enterprise or set HOT_DEV_ALLOW_OSS_FALLBACK=true to override this explicitly."
            PRO_BUILD_SUCCESS=false
            return 1
        fi

        log_warn "Pro build failed; HOT_DEV_ALLOW_OSS_FALLBACK=true so hot-dev is falling back to the OSS backend."
    fi

    if build_output=$(cd "${ROOT_DIR}" && go build -o pulse ./cmd/pulse 2>&1); then
        printf "%s\n" "${build_output}" | grep -v "^#" || true
        PRO_BUILD_SUCCESS=false
        return 0
    fi

    printf "%s\n" "${build_output}" | grep -v "^#" || true
    PRO_BUILD_SUCCESS=false
    return 1
}

# --- Configuration & Environment ---

load_env_file() {
    local env_file=$1
    if [[ -f ${env_file} ]]; then
        log_info "Loading ${env_file}"
        set +u
        set -a
        # shellcheck disable=SC1090
        source "${env_file}"
        set +a
        set -u
    fi
}

load_runtime_env_overrides() {
    local runtime_env="${PULSE_DATA_DIR}/.env"
    if [[ -f "${runtime_env}" ]]; then
        load_env_file "${runtime_env}"
        log_info "Loaded runtime .env overrides from ${runtime_env}"
    fi
}

sync_runtime_auth_env_overrides() {
    local runtime_env="${PULSE_DATA_DIR}/.env"

    if [[ "${HOT_DEV_USE_PROD_DATA:-false}" == "true" ]] || [[ "${PULSE_DATA_DIR}" == "/etc/pulse" ]]; then
        return
    fi

    hot_dev_sync_auth_env_file "${runtime_env}" "${PULSE_AUTH_USER}" "${PULSE_AUTH_PASS}"
    hot_dev_sync_audit_signing_env_file "${runtime_env}" "${PULSE_DATA_DIR}"
    hot_dev_sync_bootstrap_token_file "${PULSE_DATA_DIR}"
    log_info "Seeded managed dev auth defaults in ${runtime_env}"
}

show_startup_banner() {
    local local_frontend_url lan_frontend_url
    local_frontend_url="$(hot_dev_local_browser_url "${FRONTEND_DEV_HOST}" "${FRONTEND_DEV_PORT}")"
    lan_frontend_url="$(hot_dev_lan_browser_url "${FRONTEND_DEV_HOST}" "${FRONTEND_DEV_PORT}" "${LAN_IP}")"

    cat <<BANNER
=========================================
Starting HOT-RELOAD development mode
=========================================

Frontend: ${local_frontend_url} (Local)
BANNER
    if [[ -n "${lan_frontend_url}" ]]; then
        printf '          %s (LAN)\n' "${lan_frontend_url}"
    fi
    cat <<BANNER
Backend API: ${PULSE_DEV_API_URL}

Dev Credentials: $(hot_dev_auth_banner_line "${PULSE_AUTH_USER:-}" "${PULSE_AUTH_PASS:-}")

Mock Mode: ${PULSE_MOCK_MODE:-false}
Toggle mock mode: npm run mock:on / npm run mock:off
Mock config: npm run mock:edit

Frontend: Edit files and see changes instantly!
Backend: Auto-rebuilds when Go or embedded demo assets change!
Press Ctrl+C to stop
=========================================
BANNER
}

load_env_file "${ROOT_DIR}/.env"
load_env_file "${ROOT_DIR}/.env.local"
load_env_file "${ROOT_DIR}/.env.dev"

hot_dev_configure_network_defaults

EXTRA_CLEANUP_PORT=$((PULSE_DEV_API_PORT + 1))
HOT_DEV_RESTART_SENTINEL="${ROOT_DIR}/tmp/hot-dev.restart"
HOT_DEV_VERIFY_LOCK="${HOT_DEV_VERIFY_LOCK_FILE:-${ROOT_DIR}/tmp/hot-dev.verify.lock}"
HOT_DEV_BUILD_LOCK="${HOT_DEV_BUILD_LOCK_FILE:-${ROOT_DIR}/tmp/hot-dev.build.lock}"
HOT_DEV_SELF_BUILD_IGNORE_UNTIL_FILE="${HOT_DEV_SELF_BUILD_IGNORE_UNTIL_FILE:-${ROOT_DIR}/tmp/hot-dev.self-build-ignore-until}"
HOT_DEV_WATCHER_STARTUP_GRACE_SECONDS="${HOT_DEV_WATCHER_STARTUP_GRACE_SECONDS:-5}"
EMBEDDED_FRONTEND_DIR="${ROOT_DIR}/internal/api/frontend-modern"
EMBEDDED_FRONTEND_DIST_DIR="${EMBEDDED_FRONTEND_DIR}/dist"
BACKEND_DEBUG_LOG="${PULSE_BACKEND_LOG_FILE:-/tmp/pulse-debug.log}"

# --- Startup Checks ---

check_dependencies

kill_port() {
    local port=$1
    log_info "Cleaning up port ${port}..."
    lsof -i :"${port}" 2>/dev/null | awk 'NR>1 {print $2}' | xargs -r kill -9 2>/dev/null || true
}

process_parent_id() {
    local pid=$1
    ps -o ppid= -p "${pid}" 2>/dev/null | tr -d '[:space:]'
}

is_current_shell_descendant_of() {
    local target_pid=$1
    local current_pid=$$
    local parent_pid

    [[ -n "${target_pid}" ]] || return 1

    while [[ -n "${current_pid}" && "${current_pid}" != "1" ]]; do
        parent_pid="$(process_parent_id "${current_pid}")"
        [[ -n "${parent_pid}" && "${parent_pid}" != "${current_pid}" ]] || break
        if [[ "${parent_pid}" == "${target_pid}" ]]; then
            return 0
        fi
        current_pid="${parent_pid}"
    done

    return 1
}

kill_stale_npm_dev_wrappers() {
    local pid

    if [[ "${HOT_DEV_SKIP_NPM_CLEANUP:-false}" == "true" ]]; then
        return 0
    fi

    while IFS= read -r pid; do
        [[ -n "${pid}" ]] || continue
        [[ "${pid}" != "$$" ]] || continue
        if is_current_shell_descendant_of "${pid}"; then
            continue
        fi
        kill "${pid}" 2>/dev/null || true
    done < <(pgrep -f "npm run dev" 2>/dev/null || true)
}

log_info "Cleaning up existing processes..."

# OS-Specific Cleanup
# Note: When running under systemd (INVOCATION_ID is set), skip stopping our own service
OS_NAME=$(uname -s)
if [[ "$OS_NAME" == "Linux" ]] && [[ -z "${INVOCATION_ID:-}" ]]; then
    # Only stop pulse-dev if we're NOT running under systemd
    sudo systemctl stop pulse-dev 2>/dev/null || true
fi

pkill -f "backend-watch.sh" 2>/dev/null || true
# Only kill vite/npm processes that look like ours (simple check)
pkill -f "vite" 2>/dev/null || true
kill_stale_npm_dev_wrappers

pkill -x "pulse" 2>/dev/null || true
sleep 1
pkill -9 -x "pulse" 2>/dev/null || true


kill_port "${FRONTEND_DEV_PORT}"
kill_port "${PULSE_DEV_API_PORT}"
kill_port "${EXTRA_CLEANUP_PORT}"

# Truncate debug log
mkdir -p "$(dirname "${BACKEND_DEBUG_LOG}")"
:> "${BACKEND_DEBUG_LOG}"
BACKEND_STARTED_AT_FILE="${HOT_DEV_BACKEND_STARTED_AT_FILE:-${ROOT_DIR}/tmp/hot-dev.backend.started-at}"
mkdir -p "$(dirname "${BACKEND_STARTED_AT_FILE}")"

sleep 2

# Verify ports are free
set +o pipefail
for port in "${FRONTEND_DEV_PORT}" "${PULSE_DEV_API_PORT}"; do
    if lsof -i :"${port}" 2>/dev/null | grep -q LISTEN; then
        log_error "Port ${port} is still in use after cleanup!"
        kill_port "${port}"
        sleep 2
        if lsof -i :"${port}" 2>/dev/null | grep -q LISTEN; then
            log_error "FATAL: Cannot free port ${port}. Please manually kill the process:"
            lsof -i :"${port}"
            exit 1
        fi
    fi
done
set -o pipefail

log_info "Ports are clean!"

mkdir -p "$(dirname "${HOT_DEV_RESTART_SENTINEL}")"
if [[ ! -e "${HOT_DEV_RESTART_SENTINEL}" ]]; then
    : > "${HOT_DEV_RESTART_SENTINEL}"
fi

# Self-restart check
check_script_change() {
    local current_mtime
    current_mtime=$(stat -c %Y "${SCRIPT_PATH}" 2>/dev/null || stat -f %m "${SCRIPT_PATH}")
    if [[ "${current_mtime}" != "${SCRIPT_MTIME}" ]]; then
        log_warn "hot-dev.sh script changed! Restarting..."
        exec "$0" "$@"
    fi
}

# --- Config Setup ---

if [[ -f /etc/pulse/.env ]] && [[ -r /etc/pulse/.env ]]; then
    load_env_file "/etc/pulse/.env"
    echo "Auth configuration loaded from /etc/pulse/.env"
fi

# --- Start Backend ---

log_info "Starting backend on port ${PULSE_DEV_API_PORT}..."
cd "${ROOT_DIR}"

mkdir -p internal/api/frontend-modern/dist
touch internal/api/frontend-modern/dist/index.html

# Check if Pro module is available and use it for full audit logging support.
# Use PULSE_REPOS_DIR env var or default to the parent directory that contains sibling repos.
if ! build_backend_binary; then
    exit 1
fi

FRONTEND_PORT=${PULSE_DEV_API_PORT}
PORT=${PULSE_DEV_API_PORT}
PULSE_DEV=true  # Enable development mode features (needed for admin bypass etc)
ALLOW_ADMIN_BYPASS=1  # Allow X-Admin-Bypass header in dev mode
LOG_LEVEL="${LOG_LEVEL:-info}"
PULSE_DEV_DISABLE_BACKGROUND_AI="${PULSE_DEV_DISABLE_BACKGROUND_AI:-true}"

# Managed dev credentials default to admin/adminadminadmin unless
# HOT_DEV_AUTH_USER / HOT_DEV_AUTH_PASS explicitly override them.
PULSE_AUTH_USER="$(hot_dev_resolve_auth_user)"
PULSE_AUTH_PASS="$(hot_dev_resolve_auth_pass)"
export FRONTEND_PORT PULSE_DEV_API_PORT PORT PULSE_DEV ALLOW_ADMIN_BYPASS PULSE_AUTH_USER PULSE_AUTH_PASS LOG_LEVEL PULSE_DEV_DISABLE_BACKGROUND_AI
export PULSE_DEV_LAN PULSE_DEV_LAB_AGENTS BIND_ADDRESS ALLOWED_ORIGINS
export PULSE_ENABLE_PROXMOX_GUEST_DOCKER_DETECTION PULSE_ENABLE_PROXMOX_GUEST_DOCKER_INVENTORY PULSE_PROXMOX_GUEST_DOCKER_INVENTORY_VMIDS
export HOT_DEV_LAB_MODE_FILE

# Data Directory Setup
# Mock and real mode use SEPARATE data directories so switching between them
# (scripts/toggle-mock.sh on|off) is clean in both directions: mock data never
# leaks into the real connection ledger, and real connections/alerts never show
# up under mock. The canonical PULSE_MOCK_MODE flag is persisted to
# tmp/dev-config/.env by toggle-mock, so read it before choosing the directory.
if [[ -n ${PULSE_DATA_DIR:-} ]]; then
    log_info "Using preconfigured data directory: ${PULSE_DATA_DIR}"
elif [[ ${HOT_DEV_USE_PROD_DATA:-false} == "true" ]]; then
    export PULSE_DATA_DIR=/etc/pulse
    log_info "HOT_DEV_USE_PROD_DATA=true – using production data directory: ${PULSE_DATA_DIR}"
else
    # Read the canonical mock flag from tmp/dev-config/.env, which toggle-mock
    # writes authoritatively on every switch. A stale PULSE_MOCK_MODE exported
    # into the environment (e.g. inherited by the hot-dev supervisor from a prior
    # mock run) must NOT win here, or real mode would keep reading the mock dir.
    HOT_DEV_MOCK_FLAG=""
    if [[ -f "${ROOT_DIR}/tmp/dev-config/.env" ]]; then
        HOT_DEV_MOCK_FLAG="$(grep -E '^[[:space:]]*PULSE_MOCK_MODE=' "${ROOT_DIR}/tmp/dev-config/.env" | tail -1 | cut -d= -f2 | tr -dc 'a-z' || true)"
    fi
    if [[ -z "${HOT_DEV_MOCK_FLAG}" ]]; then
        HOT_DEV_MOCK_FLAG="${PULSE_MOCK_MODE:-}"
    fi
    if [[ "${HOT_DEV_MOCK_FLAG}" == "true" ]]; then
        export PULSE_DATA_DIR="${ROOT_DIR}/tmp/mock-data"
        mkdir -p "$PULSE_DATA_DIR"
        # The mock data dir holds no real history, so the backend may safely
        # backfill mock metrics history into its metrics.db (powers reports
        # and long-range charts in mock mode).
        export PULSE_MOCK_SEED_METRICS_STORE=true
        log_info "Mock mode: using isolated mock data directory: ${PULSE_DATA_DIR}"
    else
        DEV_CONFIG_DIR="${ROOT_DIR}/tmp/dev-config"
        mkdir -p "$DEV_CONFIG_DIR"
        export PULSE_DATA_DIR="${DEV_CONFIG_DIR}"
        log_info "Using dev config directory: ${PULSE_DATA_DIR}"
    fi
fi

sync_runtime_auth_env_overrides
load_runtime_env_overrides
hot_dev_reconcile_agent_bind_address

if [[ "${PULSE_DATA_DIR}" == "${ROOT_DIR}/tmp/dev-config" ]] && [[ ${PULSE_MOCK_MODE:-false} != "true" ]]; then
    echo "Syncing production configuration..."
    DEV_DIR="${ROOT_DIR}/tmp/dev-config" "${ROOT_DIR}/scripts/sync-production-config.sh"
fi

if [[ ${PULSE_MOCK_MODE:-false} == "true" ]]; then
    log_info "Mock mode enabled (isolated data directory: ${PULSE_DATA_DIR})"
    TOTAL_GUESTS=$((${PULSE_MOCK_NODES:-3} * (${PULSE_MOCK_VMS_PER_NODE:-3} + ${PULSE_MOCK_LXCS_PER_NODE:-3})))
    echo "Mock mode ENABLED with ${PULSE_MOCK_NODES:-3} nodes (${TOTAL_GUESTS} total guests)"
fi

show_startup_banner

# Auto-restore encryption key from backup if missing
if [[ ! -f "${PULSE_DATA_DIR}/.encryption.key" ]]; then
    BACKUP_KEY=$(find "${PULSE_DATA_DIR}" -maxdepth 1 -name '.encryption.key.bak*' -type f 2>/dev/null | head -1)
    if [[ -n "${BACKUP_KEY}" ]] && [[ -f "${BACKUP_KEY}" ]]; then
        echo ""
        log_error "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
        log_error "!! ENCRYPTION KEY WAS MISSING - AUTO-RESTORING FROM BACKUP !!"
        log_error "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
        log_error "!! Backup used: ${BACKUP_KEY}"
        log_error "!! "
        log_error "!! To find out what deleted the key:"
        log_error "!!   Linux: sudo journalctl -u encryption-key-watcher -n 100"
        log_error "!!   macOS: check /tmp/pulse-debug.log"
        log_error "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
        echo ""
        cp -f "${BACKUP_KEY}" "${PULSE_DATA_DIR}/.encryption.key"
        chmod 600 "${PULSE_DATA_DIR}/.encryption.key"
        log_info "Restored encryption key from backup"
    fi
fi

if [[ -z ${PULSE_ENCRYPTION_KEY:-} ]]; then
    if [[ -f "${PULSE_DATA_DIR}/.encryption.key" ]]; then
        export PULSE_ENCRYPTION_KEY="$(<"${PULSE_DATA_DIR}/.encryption.key")"
        log_info "Loaded encryption key from ${PULSE_DATA_DIR}/.encryption.key"
    elif [[ ${PULSE_DATA_DIR} == "${ROOT_DIR}/tmp/dev-config" ]]; then
        DEV_KEY_FILE="${PULSE_DATA_DIR}/.encryption.key"
        if [[ ! -f "${DEV_KEY_FILE}" ]]; then
            mapfile -t ENCRYPTED_FILES < <(detect_encrypted_files "${PULSE_DATA_DIR}")
            if [[ ${#ENCRYPTED_FILES[@]} -gt 0 ]]; then
                log_error "Encryption key is missing but encrypted data exists."
                log_error "Restore ${DEV_KEY_FILE} from backup before continuing."
                log_error "Encrypted files: ${ENCRYPTED_FILES[*]}"
                exit 1
            fi
            openssl rand -base64 32 > "${DEV_KEY_FILE}"
            chmod 600 "${DEV_KEY_FILE}"
            log_info "Generated dev encryption key at ${DEV_KEY_FILE}"
        fi
        export PULSE_ENCRYPTION_KEY="$(<"${DEV_KEY_FILE}")"
    elif [[ ${HOT_DEV_USE_PROD_DATA:-false} == "true" ]]; then
        # Production data mode but no key - generate one to prevent orphaned encrypted data
        mapfile -t ENCRYPTED_FILES < <(detect_encrypted_files "${PULSE_DATA_DIR}")
        if [[ ${#ENCRYPTED_FILES[@]} -gt 0 ]]; then
            log_error "Encryption key is missing but encrypted data exists."
            log_error "Restore ${PULSE_DATA_DIR}/.encryption.key from backup before continuing."
            log_error "Encrypted files: ${ENCRYPTED_FILES[*]}"
            exit 1
        fi
        log_warn "No encryption key found for ${PULSE_DATA_DIR}. Generating new key..."
        openssl rand -base64 32 > "${PULSE_DATA_DIR}/.encryption.key"
        chmod 600 "${PULSE_DATA_DIR}/.encryption.key"
        export PULSE_ENCRYPTION_KEY="$(<"${PULSE_DATA_DIR}/.encryption.key")"
        log_info "Generated new encryption key at ${PULSE_DATA_DIR}/.encryption.key"
    else
        log_warn "No encryption key found for ${PULSE_DATA_DIR}. Encrypted config may fail to load."
    fi
fi

# Set audit dir for Pro features (must be after PULSE_DATA_DIR is set)
if [[ ${PRO_BUILD_SUCCESS:-false} == "true" ]]; then
    export PULSE_AUDIT_DIR="${PULSE_DATA_DIR}"
    log_info "Pro audit logging enabled (SQLite storage in ${PULSE_AUDIT_DIR})"
fi

STARTED_BACKEND_PID=""
mark_backend_startup_grace() {
    date +%s > "${BACKEND_STARTED_AT_FILE}"
}

start_backend_process() {
    mark_backend_startup_grace
    LOG_LEVEL="${LOG_LEVEL:-info}" \
    PULSE_DEV_DISABLE_BACKGROUND_AI="${PULSE_DEV_DISABLE_BACKGROUND_AI:-true}" \
    FRONTEND_PORT="${PULSE_DEV_API_PORT:-7655}" \
    PORT="${PULSE_DEV_API_PORT:-7655}" \
    PULSE_DATA_DIR="${PULSE_DATA_DIR:-}" \
    PULSE_ENCRYPTION_KEY="${PULSE_ENCRYPTION_KEY:-}" \
    ALLOW_ADMIN_BYPASS="${ALLOW_ADMIN_BYPASS:-1}" \
    PULSE_DEV="${PULSE_DEV:-true}" \
    PULSE_DEV_LAN="${PULSE_DEV_LAN:-}" \
    PULSE_DEV_LAB_AGENTS="${PULSE_DEV_LAB_AGENTS:-}" \
    BIND_ADDRESS="${BIND_ADDRESS:-}" \
    ALLOWED_ORIGINS="${ALLOWED_ORIGINS:-}" \
    PULSE_AUTH_USER="${PULSE_AUTH_USER:-}" \
    PULSE_AUTH_PASS="${PULSE_AUTH_PASS:-}" \
    PULSE_ENABLE_PROXMOX_GUEST_DOCKER_DETECTION="${PULSE_ENABLE_PROXMOX_GUEST_DOCKER_DETECTION:-}" \
    PULSE_ENABLE_PROXMOX_GUEST_DOCKER_INVENTORY="${PULSE_ENABLE_PROXMOX_GUEST_DOCKER_INVENTORY:-}" \
    PULSE_PROXMOX_GUEST_DOCKER_INVENTORY_VMIDS="${PULSE_PROXMOX_GUEST_DOCKER_INVENTORY_VMIDS:-}" \
    ALLOWED_ORIGINS="${ALLOWED_ORIGINS:-}" \
    LOG_FILE="${BACKEND_DEBUG_LOG}" \
    LOG_MAX_SIZE="50" \
    HTTP_PROXY="" HTTPS_PROXY="" http_proxy="" https_proxy="" \
    NO_PROXY="" no_proxy="" \
    ./pulse </dev/null >> "${BACKEND_DEBUG_LOG}" 2>&1 &
    STARTED_BACKEND_PID=$!
}

start_backend_process
BACKEND_PID="${STARTED_BACKEND_PID}"

sleep 2

if ! kill -0 "${BACKEND_PID}" 2>/dev/null; then
    log_error "Backend failed to start! Check ${BACKEND_DEBUG_LOG}"
    exit 1
fi

# --- Backend Health Monitor ---
# Restarts Pulse if it dies unexpectedly (not from file watcher rebuild),
# enforces single-instance, and detects the alive-but-unresponsive state
# (process exists but /api/health is not serving) which a process-only check
# blind-spots. After backend startup grace, consecutive misses on /api/health
# -> kill and restart.
log_info "Starting backend health monitor..."
(
    UNHEALTHY_STREAK=0
    UNHEALTHY_THRESHOLD="${HOT_DEV_BACKEND_UNHEALTHY_THRESHOLD:-4}"
    BACKEND_HEALTH_STARTUP_GRACE_SECONDS="${HOT_DEV_BACKEND_HEALTH_STARTUP_GRACE_SECONDS:-180}"
    BACKEND_PROCESS_MISSING_GRACE_SECONDS="${HOT_DEV_BACKEND_PROCESS_MISSING_GRACE_SECONDS:-10}"

    [[ "${UNHEALTHY_THRESHOLD}" =~ ^[1-9][0-9]*$ ]] || UNHEALTHY_THRESHOLD=4
    [[ "${BACKEND_HEALTH_STARTUP_GRACE_SECONDS}" =~ ^[0-9]+$ ]] || BACKEND_HEALTH_STARTUP_GRACE_SECONDS=180
    [[ "${BACKEND_PROCESS_MISSING_GRACE_SECONDS}" =~ ^[0-9]+$ ]] || BACKEND_PROCESS_MISSING_GRACE_SECONDS=10

    backend_serving() {
        curl -sf -o /dev/null --max-time 3 \
            "http://127.0.0.1:${PULSE_DEV_API_PORT:-7655}/api/health" 2>/dev/null
    }

    backend_startup_age_seconds() {
        local backend_started_at
        local now

        [[ -r "${BACKEND_STARTED_AT_FILE}" ]] || return 1
        backend_started_at="$(<"${BACKEND_STARTED_AT_FILE}")"
        [[ "${backend_started_at}" =~ ^[0-9]+$ ]] || return 1

        now="$(date +%s)"
        printf "%s\n" "$((now - backend_started_at))"
    }

    backend_in_startup_grace() {
        local age

        age="$(backend_startup_age_seconds)" || return 1
        (( age < BACKEND_HEALTH_STARTUP_GRACE_SECONDS ))
    }

    while true; do
        sleep 5
        PULSE_COUNT="$(hot_dev_pulse_process_count)"

        if [[ "$PULSE_COUNT" -eq 0 ]]; then
            startup_age="$(backend_startup_age_seconds || printf '%s\n' "${BACKEND_PROCESS_MISSING_GRACE_SECONDS}")"
            if backend_in_startup_grace && (( startup_age < BACKEND_PROCESS_MISSING_GRACE_SECONDS )); then
                UNHEALTHY_STREAK=0
                log_warn "⚠️  Pulse process not running yet during backend startup grace (${startup_age}/${BACKEND_PROCESS_MISSING_GRACE_SECONDS}s)"
                continue
            fi
            log_warn "⚠️  Pulse died unexpectedly, restarting..."
            start_backend_process
            NEW_PID="${STARTED_BACKEND_PID}"
            sleep 2
            if kill -0 "$NEW_PID" 2>/dev/null; then
                log_info "✓ Backend auto-restarted (PID: $NEW_PID)"
            else
                log_error "✗ Backend failed to auto-restart!"
            fi
            UNHEALTHY_STREAK=0
        elif [[ "$PULSE_COUNT" -gt 1 ]]; then
            log_error "⚠️  Multiple Pulse processes detected ($PULSE_COUNT), killing all and restarting..."
            pkill -9 -f "^\./pulse$" 2>/dev/null || true
            sleep 2
            start_backend_process
            NEW_PID="${STARTED_BACKEND_PID}"
            sleep 2
            if kill -0 "$NEW_PID" 2>/dev/null; then
                log_info "✓ Backend restarted after killing duplicates (PID: $NEW_PID)"
            else
                log_error "✗ Backend failed to restart after killing duplicates!"
            fi
            UNHEALTHY_STREAK=0
        elif ! backend_serving; then
            if backend_in_startup_grace; then
                UNHEALTHY_STREAK=0
                log_warn "⚠️  Pulse /api/health not serving yet during backend startup grace (${BACKEND_HEALTH_STARTUP_GRACE_SECONDS}s)"
                continue
            fi
            UNHEALTHY_STREAK=$((UNHEALTHY_STREAK + 1))
            log_warn "⚠️  Pulse alive but /api/health unresponsive (streak ${UNHEALTHY_STREAK}/${UNHEALTHY_THRESHOLD})"
            if [[ "$UNHEALTHY_STREAK" -ge "$UNHEALTHY_THRESHOLD" ]]; then
                log_error "⚠️  Killing unresponsive Pulse and restarting..."
                pkill -9 -f "^\./pulse$" 2>/dev/null || true
                sleep 2
                start_backend_process
                NEW_PID="${STARTED_BACKEND_PID}"
                sleep 2
                if kill -0 "$NEW_PID" 2>/dev/null; then
                    log_info "✓ Backend restarted after unresponsive state (PID: $NEW_PID)"
                else
                    log_error "✗ Backend failed to restart after unresponsive state!"
                fi
                UNHEALTHY_STREAK=0
            fi
        else
            UNHEALTHY_STREAK=0
        fi
    done
) &
HEALTH_MONITOR_PID=$!

# --- File Watcher ---

log_info "Starting backend file watcher..."
(
    cd "${ROOT_DIR}"
    
    WATCHER_READY_AT=$(date +%s)
    LAST_RESTART_TIME=0
    SELF_BUILD_IGNORE_UNTIL=$((WATCHER_READY_AT + HOT_DEV_WATCHER_STARTUP_GRACE_SECONDS + 5))
    LAST_RESTART_SENTINEL_MARKER=""
    LAST_PULSE_BINARY_MARKER=""

    watcher_within_startup_grace() {
        local now
        now=$(date +%s)
        (( now - WATCHER_READY_AT < HOT_DEV_WATCHER_STARTUP_GRACE_SECONDS ))
    }

    mark_self_build_output() {
        local now
        now=$(date +%s)
        SELF_BUILD_IGNORE_UNTIL=$((now + 5))
        mkdir -p "$(dirname "${HOT_DEV_SELF_BUILD_IGNORE_UNTIL_FILE}")"
        printf '%s\n' "${SELF_BUILD_IGNORE_UNTIL}" > "${HOT_DEV_SELF_BUILD_IGNORE_UNTIL_FILE}"
    }

    manual_build_event_suppressed() {
        local now
        local shared_ignore_until

        if build_lock_active; then
            return 0
        fi

        now=$(date +%s)
        shared_ignore_until=""
        if [[ -r "${HOT_DEV_SELF_BUILD_IGNORE_UNTIL_FILE}" ]]; then
            shared_ignore_until="$(<"${HOT_DEV_SELF_BUILD_IGNORE_UNTIL_FILE}")"
        fi
        if [[ "${shared_ignore_until}" =~ ^[0-9]+$ ]] && (( now < shared_ignore_until )); then
            return 0
        fi

        (( now < SELF_BUILD_IGNORE_UNTIL ))
    }

    should_rebuild_backend_for_change() {
        local changed_file=$1

        if [[ "${changed_file}" == "${EMBEDDED_FRONTEND_DIST_DIR}" ]] || [[ "${changed_file}" == "${EMBEDDED_FRONTEND_DIST_DIR}/"* ]]; then
            return 0
        fi

        [[ "${changed_file}" == *.go ]] || return 1
        [[ "${changed_file}" == *_test.go ]] && return 1

        return 0
    }

    file_event_marker() {
        local file_path=$1

        [[ -e "${file_path}" ]] || return 1

        if stat -f '%m:%z' "${file_path}" >/dev/null 2>&1; then
            stat -f '%m:%z' "${file_path}"
        else
            stat -c '%Y:%s' "${file_path}"
        fi
    }

    LAST_PULSE_BINARY_MARKER="$(file_event_marker "${ROOT_DIR}/pulse" || true)"

    build_lock_active() {
        local owner_pid

        [[ -f "${HOT_DEV_BUILD_LOCK}" ]] || return 1

        owner_pid="$(awk -F= '/^pid=/{print $2; exit}' "${HOT_DEV_BUILD_LOCK}" 2>/dev/null || true)"
        if [[ -z "${owner_pid}" ]]; then
            rm -f "${HOT_DEV_BUILD_LOCK}"
            return 1
        fi

        if ! kill -0 "${owner_pid}" 2>/dev/null; then
            rm -f "${HOT_DEV_BUILD_LOCK}"
            return 1
        fi

        return 0
    }

    set_build_lock() {
        mkdir -p "$(dirname "${HOT_DEV_BUILD_LOCK}")"
        printf 'pid=%s\ncreated_at=%s\n' "${BASHPID:-$$}" "$(date -u +"%Y-%m-%dT%H:%M:%SZ")" > "${HOT_DEV_BUILD_LOCK}"
    }

    clear_build_lock() {
        rm -f "${HOT_DEV_BUILD_LOCK}"
    }

    verify_lock_active() {
        local owner_pid

        [[ -f "${HOT_DEV_VERIFY_LOCK}" ]] || return 1

        owner_pid="$(awk -F= '/^pid=/{print $2; exit}' "${HOT_DEV_VERIFY_LOCK}" 2>/dev/null || true)"
        if [[ -z "${owner_pid}" ]]; then
            rm -f "${HOT_DEV_VERIFY_LOCK}"
            return 1
        fi

        if ! kill -0 "${owner_pid}" 2>/dev/null; then
            rm -f "${HOT_DEV_VERIFY_LOCK}"
            return 1
        fi

        return 0
    }

    restart_backend() {
        # Debounce: skip if we restarted less than 3 seconds ago
        local now
        now=$(date +%s)
        if (( now - LAST_RESTART_TIME < 3 )); then
            return
        fi
        LAST_RESTART_TIME=$now
        mark_backend_startup_grace
        mark_self_build_output

        log_info "Restarting backend..."

        # Re-source the canonical runtime .env so mock-mode changes take
        # effect without a full hot-dev restart.
        sync_runtime_auth_env_overrides
        load_runtime_env_overrides
        hot_dev_reconcile_agent_bind_address

        # Kill ALL pulse processes (not just one) to prevent duplicates
        pkill -f "^\./pulse$" 2>/dev/null || true
        sleep 1
        # Force kill any remaining
        pkill -9 -f "^\./pulse$" 2>/dev/null || true
        sleep 1

        # Close inherited stdin (</dev/null) to prevent pipe fd leaks when
        # this function is called from inside a piped while-read loop.
        start_backend_process
        NEW_PID="${STARTED_BACKEND_PID}"
        sleep 1

        # Verify exactly one process is running
        PULSE_COUNT="$(hot_dev_pulse_process_count)"
        if [[ "$PULSE_COUNT" -eq 1 ]] && kill -0 "$NEW_PID" 2>/dev/null; then
            log_info "✓ Backend restarted (PID: $NEW_PID)"
        elif [[ "$PULSE_COUNT" -gt 1 ]]; then
            log_error "✗ Multiple processes after restart ($PULSE_COUNT) - killing all"
            pkill -9 -f "^\./pulse$" 2>/dev/null || true
        else
            log_error "✗ Backend failed to start! Check ${BACKEND_DEBUG_LOG}"
        fi
    }

    LAST_REBUILD_TIME=0

    rebuild_backend() {
        local changed_file=$1

        if build_lock_active; then
            log_info "Managed build already in progress; skipping duplicate rebuild event."
            return
        fi

        # Debounce: skip if we rebuilt less than 2 seconds ago (batch saves)
        local now
        now=$(date +%s)
        if (( now - LAST_REBUILD_TIME < 2 )); then
            return
        fi
        LAST_REBUILD_TIME=$now

        echo ""
        log_info "🔄 Change detected: $(basename "$changed_file")"
        log_info "Rebuilding backend..."

        # Suppress manual pulse-binary restart handling for the build output
        # generated by this managed rebuild before the compiler touches ./pulse.
        mark_self_build_output
        set_build_lock

        # Use the same build logic as the initial build.
        if build_backend_binary; then
            mark_self_build_output
            restart_backend
            clear_build_lock
        else
            clear_build_lock
            log_error "✗ Build failed; keeping the current backend process running."
        fi
        LAST_REBUILD_TIME=$(date +%s)
        log_info "Watching for changes..."
    }

    if command -v inotifywait >/dev/null 2>&1; then
        # Linux: inotifywait
        # Watch source directories AND the pulse binary itself (so manual builds trigger restart)
        inotifywait -r -m -e modify,create,delete,move,attrib \
            --exclude '(vendor/|node_modules/|\.git/|\.swp$|\.tmp$|~$)' \
            --format '%e %w%f' \
            "${ROOT_DIR}/cmd" "${ROOT_DIR}/internal" "${ROOT_DIR}/pkg" "${ROOT_DIR}/pulse" "${HOT_DEV_RESTART_SENTINEL}" 2>/dev/null | \
        while read -r event changed_file; do
            check_script_change "$@"
            if watcher_within_startup_grace && { [[ "$changed_file" == "${HOT_DEV_RESTART_SENTINEL}" ]] || [[ "$changed_file" == "${ROOT_DIR}/pulse" ]]; }; then
                continue
            fi
            if [[ "$changed_file" == "${HOT_DEV_RESTART_SENTINEL}" ]]; then
                event_marker="$(file_event_marker "${HOT_DEV_RESTART_SENTINEL}" || true)"
                if [[ -n "${event_marker}" && "${event_marker}" == "${LAST_RESTART_SENTINEL_MARKER}" ]]; then
                    continue
                fi
                LAST_RESTART_SENTINEL_MARKER="${event_marker}"
                log_info "🚀 Managed restart requested, restarting backend..."
                restart_backend
            elif [[ "$changed_file" == "${ROOT_DIR}/pulse" ]]; then
                if verify_lock_active; then
                    continue
                fi
                if manual_build_event_suppressed; then
                    continue
                fi
                event_marker="$(file_event_marker "${ROOT_DIR}/pulse" || true)"
                if [[ -n "${event_marker}" && "${event_marker}" == "${LAST_PULSE_BINARY_MARKER}" ]]; then
                    continue
                fi
                LAST_PULSE_BINARY_MARKER="${event_marker}"
                log_info "🚀 Manual build detected (pulse binary changed), restarting..."
                restart_backend
            elif verify_lock_active; then
                continue
            elif should_rebuild_backend_for_change "$changed_file"; then
                rebuild_backend "$changed_file"
            fi
        done
    elif command -v fswatch >/dev/null 2>&1; then
        # macOS: fswatch
        # Note: AttributeModified is needed because `touch` on macOS only fires
        # that event (not Updated), and touch is the recommended way to trigger
        # a manual rebuild.
        log_info "Using fswatch for file monitoring"

        # Watch the pulse binary too — if someone does `go build -o pulse`
        # manually, we should restart without rebuilding. Watch the embedded
        # frontend parent dir as well so atomic dist swaps trigger a backend
        # rebuild for the backend-served demo surface.
        fswatch --event Created --event Updated --event Renamed --event AttributeModified \
            "${ROOT_DIR}/pulse" "${HOT_DEV_RESTART_SENTINEL}" "${EMBEDDED_FRONTEND_DIR}" 2>/dev/null | \
        while read -r changed_file; do
            if watcher_within_startup_grace && { [[ "$changed_file" == "${HOT_DEV_RESTART_SENTINEL}" ]] || [[ "$changed_file" == "${ROOT_DIR}/pulse" ]]; }; then
                continue
            fi
            if [[ "$changed_file" == "${HOT_DEV_RESTART_SENTINEL}" ]]; then
                event_marker="$(file_event_marker "${HOT_DEV_RESTART_SENTINEL}" || true)"
                if [[ -n "${event_marker}" && "${event_marker}" == "${LAST_RESTART_SENTINEL_MARKER}" ]]; then
                    continue
                fi
                LAST_RESTART_SENTINEL_MARKER="${event_marker}"
                log_info "🚀 Managed restart requested, restarting backend..."
            elif should_rebuild_backend_for_change "$changed_file"; then
                if verify_lock_active; then
                    continue
                fi
                rebuild_backend "$changed_file"
                continue
            elif [[ "$changed_file" == "${ROOT_DIR}/pulse" ]]; then
                if verify_lock_active; then
                    continue
                fi
                if manual_build_event_suppressed; then
                    continue
                fi
                event_marker="$(file_event_marker "${ROOT_DIR}/pulse" || true)"
                if [[ -n "${event_marker}" && "${event_marker}" == "${LAST_PULSE_BINARY_MARKER}" ]]; then
                    continue
                fi
                LAST_PULSE_BINARY_MARKER="${event_marker}"
                log_info "🚀 Manual build detected (pulse binary changed), restarting..."
            else
                continue
            fi
            restart_backend
        done &
        BINARY_WATCH_PID=$!

        fswatch -r -L \
            --event Created --event Updated --event Removed --event Renamed --event AttributeModified \
            --exclude '\.git/' --exclude 'vendor/' --exclude 'node_modules/' \
            --include '\.go$' \
            "${ROOT_DIR}/cmd" "${ROOT_DIR}/internal" "${ROOT_DIR}/pkg" 2>/dev/null | \
        while read -r changed_file; do
            if verify_lock_active; then
                continue
            fi
            if should_rebuild_backend_for_change "$changed_file"; then
                rebuild_backend "$changed_file"
            fi
        done
    else
        log_warn "No supported file watcher found (inotifywait or fswatch). Auto-rebuild disabled."
        sleep 3600
    fi
) &
WATCHER_PID=$!

# --- Cleanup Handler ---

cleanup() {
    echo ""
    log_info "Stopping services..."

    # Kill Health Monitor
    if [[ -n ${HEALTH_MONITOR_PID:-} ]] && kill -0 "${HEALTH_MONITOR_PID}" 2>/dev/null; then
        kill "${HEALTH_MONITOR_PID}" 2>/dev/null || true
    fi

    # Kill Watcher
    if [[ -n ${WATCHER_PID:-} ]] && kill -0 "${WATCHER_PID}" 2>/dev/null; then
        kill "${WATCHER_PID}" 2>/dev/null || true
    fi

    # Kill Backend
    # We re-find the PID because it might have changed during restart
    CURRENT_BACKEND_PID=$(pgrep -f "^\./pulse$" || true)
    if [[ -n ${CURRENT_BACKEND_PID} ]]; then
        kill "${CURRENT_BACKEND_PID}" 2>/dev/null || true
        sleep 1
        if kill -0 "${CURRENT_BACKEND_PID}" 2>/dev/null; then
            kill -9 "${CURRENT_BACKEND_PID}" 2>/dev/null || true
        fi
    fi

    # Kill Frontend (Vite)
    if [[ -n ${VITE_PID:-} ]] && kill -0 "${VITE_PID}" 2>/dev/null; then
        kill "${VITE_PID}" 2>/dev/null || true
    fi
    
    # Kill file backup watcher
    if [[ -n ${BACKUP_WATCHER_PID:-} ]] && kill -0 "${BACKUP_WATCHER_PID}" 2>/dev/null; then
        kill "${BACKUP_WATCHER_PID}" 2>/dev/null || true
    fi

    # Fallback cleanup
    pkill -f "inotifywait.*pulse" 2>/dev/null || true
    pkill -f "fswatch.*pulse" 2>/dev/null || true
    pkill -f "watch-snapshot.sh" 2>/dev/null || true

    log_info "Hot-dev stopped."
}
trap cleanup INT TERM EXIT

# --- Start File Backup Watcher (optional) ---

SNAPSHOT_SCRIPT="${ROOT_DIR}/scripts/watch-snapshot.sh"
if [[ -x "${SNAPSHOT_SCRIPT}" ]]; then
    log_info "Starting snapshot watcher..."
    "${SNAPSHOT_SCRIPT}" > /tmp/pulse-watch-snapshot.log 2>&1 &
    BACKUP_WATCHER_PID=$!
    log_info "Snapshots: ~/.pulse-snapshots (PID: ${BACKUP_WATCHER_PID})"
fi

# --- Start Frontend ---

log_info "Starting frontend with hot-reload on port ${FRONTEND_DEV_PORT}..."
cd "${ROOT_DIR}/frontend-modern"

# Run Vite in background and wait for it, so we can trap signals properly
npx vite --config vite.config.ts --host "${FRONTEND_DEV_HOST}" --port "${FRONTEND_DEV_PORT}" --clearScreen false &
VITE_PID=$!

wait "$VITE_PID"