Skip to content

armgit / ODS

Public Imported from GitHub
0

ODS V3 Pre-Release: Public testing and refinement ahead of the official V3 launch. Turn your PC, Mac, or Linux box into a private AI server.

19.0 KB
Raw Blame
#!/bin/bash
# ODS Comprehensive Health Check
# Tests each component with actual API calls, not just connectivity
# Exit codes: 0=healthy, 1=degraded (some services down), 2=critical (core services down)
#
# Usage: ./health-check.sh [--json] [--quiet]

# ── Bash 4+ guard ─────────────────────────────────────────────────────────────
# service-registry.sh requires associative arrays (declare -A) which need Bash 4+.
# macOS ships Bash 3.2; if running there, re-exec under Homebrew bash.
if [ "${BASH_VERSINFO[0]}" -lt 4 ]; then
    for _brew_bash in /opt/homebrew/bin/bash /usr/local/bin/bash; do
        if [ -x "$_brew_bash" ] && [ "$("$_brew_bash" -c 'echo "${BASH_VERSINFO[0]}"')" -ge 4 ]; then
            exec "$_brew_bash" "$0" "$@"
        fi
    done
    echo "Error: Bash 4+ required. macOS ships Bash 3.2. Install newer bash: brew install bash" >&2
    exit 2
fi

set -euo pipefail

# Parse args
JSON_OUTPUT=false
QUIET=false
for arg in "$@"; do
    case $arg in
        --json) JSON_OUTPUT=true ;;
        --quiet) QUIET=true ;;
    esac
done
# --json is for machines: stdout must be the JSON document and nothing else,
# so the human banner and per-check lines (all routed through log) are off.
if $JSON_OUTPUT; then
    QUIET=true
fi

# Config (defaults; .env overrides after load_env_file below)
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" 
if [[ -f "$SCRIPT_DIR/lib/service-registry.sh" ]]; then 
    . "$SCRIPT_DIR/lib/service-registry.sh" 
    sr_load 
fi
INSTALL_DIR="${INSTALL_DIR:-$HOME/ods}"
LLM_HOST="${LLM_HOST:-localhost}"
TIMEOUT="${TIMEOUT:-5}"

# Safe .env loading for port overrides (no eval; use lib/safe-env.sh)
[[ -f "$SCRIPT_DIR/lib/safe-env.sh" ]] && . "$SCRIPT_DIR/lib/safe-env.sh"
load_env_file "${INSTALL_DIR}/.env"
sr_resolve_ports
# 8080 is llama-server's container port; compose publishes it on the host as
# OLLAMA_PORT (default 11434), which sr_resolve_ports has just applied.
LLM_PORT="${LLM_PORT:-${SERVICE_PORTS[llama-server]:-11434}}"

# Colors (disabled for JSON/quiet)
if $JSON_OUTPUT || $QUIET; then
    GREEN="" RED="" YELLOW="" CYAN="" NC=""
else
    GREEN='\033[0;32m' RED='\033[0;31m' YELLOW='\033[1;33m' CYAN='\033[0;36m' NC='\033[0m'
fi

# Track results (indexed arrays — Bash 3.2 compatible as defense-in-depth)
declare -a RESULT_KEYS=()
declare -a RESULT_VALS=()
CRITICAL_FAIL=false
ANY_FAIL=false

# Set a result: result_set key value
result_set() {
    local key="$1" val="$2" i
    for i in "${!RESULT_KEYS[@]}"; do
        if [[ "${RESULT_KEYS[$i]}" == "$key" ]]; then
            RESULT_VALS[i]="$val"
            return
        fi
    done
    RESULT_KEYS+=("$key")
    RESULT_VALS+=("$val")
}

# Get a result: result_get key
result_get() {
    local key="$1" i
    for i in "${!RESULT_KEYS[@]}"; do
        if [[ "${RESULT_KEYS[$i]}" == "$key" ]]; then
            echo "${RESULT_VALS[$i]}"
            return
        fi
    done
}

log() { $QUIET || echo -e "$1"; }

# Portable millisecond timestamp (macOS BSD date lacks %N)
_now_ms() {
    python3 -c 'import time; print(int(time.time() * 1000))' 2>/dev/null || echo "$(date +%s)000"
}

# ── Test functions ──────────────────────────────────────────────────────────

# llama-server: critical path — performs an actual inference test
test_llm() {
    local start
    start=$(_now_ms)
    # llama-server serves its OpenAI-compatible API under /v1. Honor
    # LLM_API_BASE_PATH from .env (written by phase 06, default /v1) so the
    # probe hits the path the configured backend serves.
    local base_path="${LLM_API_BASE_PATH:-/v1}"
    local response
    response=$(curl -sf --max-time $TIMEOUT \
        -H "Content-Type: application/json" \
        -d '{"model":"default","prompt":"Hi","max_tokens":1}' \
        "http://${LLM_HOST}:${LLM_PORT}${base_path}/completions" 2>/dev/null)
    local end
    end=$(_now_ms)

    if echo "$response" | grep -q '"text"'; then
        result_set "llm" "ok"
        result_set "llm_latency" "$((end - start))"
        return 0
    fi
    result_set "llm" "fail"
    CRITICAL_FAIL=true
    ANY_FAIL=true
    return 1
}

# Check Docker container state for a service
# Returns: container state string (or empty if docker unavailable/container name missing)
check_container_state() {
    local sid="$1"
    local container="${SERVICE_CONTAINERS[$sid]}"

    # Guard: empty container name
    [[ -z "$container" ]] && return 0

    # Skip if docker not available
    if ! command -v docker &>/dev/null; then
        return 0
    fi

    # Get container state via docker inspect
    local state
    state=$(docker inspect --format '{{.State.Status}}' "$container" 2>&1)
    local inspect_exit=$?

    if [[ $inspect_exit -ne 0 ]]; then
        echo "not_found"
        return 1
    elif [[ "$state" == "running" ]]; then
        echo "running"
        return 0
    elif [[ "$state" == "restarting" ]]; then
        echo "restarting"
        return 1
    else
        # exited, paused, dead, created
        echo "$state"
        return 1
    fi
}

# Generic registry-driven service health check
test_service() {
    local sid="$1"
    local port_env="${SERVICE_PORT_ENVS[$sid]}"
    local default_port="${SERVICE_PORTS[$sid]}"
    local health="${SERVICE_HEALTH[$sid]}"
    local timeout="${SERVICE_HEALTH_TIMEOUTS[$sid]:-$TIMEOUT}"

    # Resolve port
    local port="$default_port"
    [[ -n "$port_env" ]] && port="${!port_env:-$default_port}"

    if [[ "${SERVICE_HEALTH_SOURCES[$sid]:-http}" == "container" ]]; then
        # A stopped container can retain its last healthy result. Require
        # both current state and health; missing Docker/containers fail too.
        local inspection state container_health
        inspection=$(docker inspect --format '{{.State.Status}} {{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' \
            "${SERVICE_CONTAINERS[$sid]}" 2>/dev/null) || return 1
        read -r state container_health <<< "$inspection"
        [[ "$state" == "running" && "$container_health" == "healthy" ]]
        return
    fi

    [[ -z "$health" || "$port" == "0" ]] && return 1

    # Check container state first (if docker available)
    local container_state
    container_state=$(check_container_state "$sid")
    if [[ -n "$container_state" && "$container_state" != "running" ]]; then
        result_set "$sid" "fail"
        ANY_FAIL=true
        return 1
    fi

    if curl -sf --max-time "$timeout" "http://127.0.0.1:${port}${health}" >/dev/null 2>&1; then
        result_set "$sid" "ok"
        return 0
    fi
    result_set "$sid" "fail"
    ANY_FAIL=true
    return 1
}

# System-level: GPU
test_gpu() {
    if command -v nvidia-smi &>/dev/null; then
        local gpu_info
        gpu_info=$(nvidia-smi --query-gpu=memory.used,memory.total,utilization.gpu,temperature.gpu --format=csv,noheader,nounits 2>/dev/null | head -1)
        if [ -n "$gpu_info" ]; then
            IFS=',' read -r mem_used mem_total gpu_util temp <<< "$gpu_info"
            result_set "gpu" "ok"
            result_set "gpu_mem_used" "${mem_used// /}"
            result_set "gpu_mem_total" "${mem_total// /}"
            result_set "gpu_util" "${gpu_util// /}"
            result_set "gpu_temp" "${temp// /}"

            # Warn if GPU memory > 95% (approaching OOM) or temp > 80C.
            # Deliberately NOT based on utilization: a llama-server doing
            # inference legitimately pins the GPU at ~100% util, so a
            # util-based warning would fire during normal, healthy load.
            local mem_used mem_total
            mem_used="$(result_get "gpu_mem_used")"
            mem_total="$(result_get "gpu_mem_total")"
            if [[ "$mem_used" =~ ^[0-9]+$ && "$mem_total" =~ ^[0-9]+$ ]] \
                && [ "$mem_total" -gt 0 ] \
                && [ $(( mem_used * 100 / mem_total )) -gt 95 ]; then
                result_set "gpu" "warn"
            fi
            if [ "$(result_get "gpu_temp")" -gt 80 ] 2>/dev/null; then
                result_set "gpu" "warn"
            fi
            return 0
        fi
    fi
    result_set "gpu" "unavailable"
    return 1
}

# System-level: Disk
test_disk() {
    local usage
    # df -P forces POSIX single-line output. Plain `df -h` wraps a long device
    # name onto a second line, which shifts awk's column indexes so `$5` reads
    # the mount point ("/") instead of the capacity — that then trips the
    # numeric comparison below. Grab the capacity field by its trailing "%" so
    # the parse survives any column shift, then strip the percent sign.
    usage=$(df -P "$INSTALL_DIR" 2>/dev/null \
        | awk 'NR>1 { for (i = 1; i <= NF; i++) if ($i ~ /%$/) { gsub(/%/, "", $i); print $i; exit } }')
    # Only treat the probe as successful when we parsed a numeric percentage;
    # this also keeps the `-gt` comparison from erroring on unexpected output.
    if [[ "$usage" =~ ^[0-9]+$ ]]; then
        result_set "disk" "ok"
        result_set "disk_usage" "$usage"
        if [ "$usage" -gt 90 ]; then
            result_set "disk" "warn"
        fi
        return 0
    fi
    result_set "disk" "unavailable"
    return 1
}

# Helper: run test_service for a service ID and log the result
check_service() {
    local sid="$1"
    local name="${SERVICE_NAMES[$sid]:-$sid}"
    if test_service "$sid" 2>/dev/null; then
        log "  ${GREEN}✓${NC} $name - healthy"
        return 0
    else
        log "  ${YELLOW}!${NC} $name - not responding"
        return 1
    fi
}

# Helper: run test_service in background and store result in temp file
check_service_async() {
    local sid="$1"
    local result_file="$2"

    # Check container state first. check_container_state exits non-zero for
    # any not-running state; without the guard, set -e kills this background
    # subshell before the result file is written and the service silently
    # disappears from the report.
    local container_state
    container_state=$(check_container_state "$sid") || true

    if test_service "$sid" 2>/dev/null; then
        echo "ok:$sid:$container_state" > "$result_file"
    else
        echo "fail:$sid:$container_state" > "$result_file"
    fi
}

# Read selected fragments without changing the installation. Disabled-named
# fragments may be selected by native Pixel, even if their container is gone.
declare -a SELECTED_FRAGMENTS=()
resolve_health_selection() {
    local selection fragments fragment
    selection=$(bash "$SCRIPT_DIR/scripts/resolve-compose-stack.sh" \
        --script-dir "$INSTALL_DIR" --tier "${TIER:-1}" \
        --gpu-backend "${GPU_BACKEND:-nvidia}" --gpu-count "${GPU_COUNT:-1}" \
        --ods-mode "${ODS_MODE:-local}" --selection-json) || return 1
    fragments=$(printf '%s' "$selection" | python3 -c '
import json, sys
value = json.load(sys.stdin)
items = value.get("fragments") if isinstance(value, dict) else None
if not isinstance(items, list) or not items or any(
    not isinstance(item, str) or not item or "\n" in item or "\r" in item
    for item in items
):
    raise SystemExit("Invalid selected Compose fragments")
print("\n".join(items))
') || return 1
    while IFS= read -r fragment; do
        [[ "$fragment" == /* ]] || fragment="$INSTALL_DIR/$fragment"
        SELECTED_FRAGMENTS+=("$fragment")
    done <<< "$fragments"
}

# This script probes host HTTP and declared container healthchecks, not the
# socket-only/native agent surfaces. A missing selected service still fails.
core_service_not_probed() {
    local sid="$1" service_dir="$INSTALL_DIR/extensions/services/$1" fragment
    [[ "$sid" == "open-webui" && "${ENABLE_OPEN_WEBUI:-true}" == "false" ]] && return 0
    if [[ "${SERVICE_PORTS[$sid]}" == "0" && "${SERVICE_HEALTH_SOURCES[$sid]:-http}" != "container" ]]; then
        return 0
    fi
    [[ -f "$service_dir/compose.yaml.disabled" && ! -f "$service_dir/compose.yaml" ]] || return 1
    # Unknown selection is not permission to skip a potentially active core
    # service. The caller also records the selection error as critical.
    $HEALTH_SELECTION_OK || return 1
    for fragment in "${SELECTED_FRAGMENTS[@]}"; do
        [[ "$fragment" -ef "$service_dir/compose.yaml.disabled" ]] && return 1
    done
    return 0
}

# ── Run tests ───────────────────────────────────────────────────────────────

# Create temp dir for parallel results
TEMP_DIR=$(mktemp -d)
trap 'rm -rf "$TEMP_DIR"' EXIT

log "${CYAN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
log "${CYAN}  ODS Health Check${NC}"
log "${CYAN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
log ""

log "${CYAN}Core Services:${NC}"

HEALTH_SELECTION_OK=false
if resolve_health_selection; then
    HEALTH_SELECTION_OK=true
else
    result_set "stack_selection" "fail"
    CRITICAL_FAIL=true
    ANY_FAIL=true
    log "  ${RED}✗${NC} Could not determine the selected stack; repair the installation before trusting its health."
fi

# llama-server (critical — does inference test, not just health)
if test_llm 2>/dev/null; then
    log "  ${GREEN}✓${NC} llama-server - inference working ($(result_get "llm_latency")ms)"
else
    log "  ${RED}✗${NC} llama-server - CRITICAL: inference failed"
fi

# Launch all other core services in parallel
declare -a CORE_PIDS=()
declare -a CORE_SIDS=()
for sid in "${SERVICE_IDS[@]}"; do
    [[ "$sid" == "llama-server" ]] && continue
    [[ "${SERVICE_CATEGORIES[$sid]}" != "core" ]] && continue
    core_service_not_probed "$sid" && continue
    result_file="$TEMP_DIR/core_$sid"
    check_service_async "$sid" "$result_file" &
    CORE_PIDS+=($!)
    CORE_SIDS+=("$sid")
done

# Wait for all core service checks to complete
for pid in "${CORE_PIDS[@]}"; do
    wait "$pid" 2>/dev/null || true
done

# Display core service results. The async checks run in subshells, so their
# result_set/ANY_FAIL mutations are lost — the parent must aggregate status
# from the result files or failures never reach the summary, exit code, or
# JSON output.
for sid in "${CORE_SIDS[@]}"; do
    result_file="$TEMP_DIR/core_$sid"
    name="${SERVICE_NAMES[$sid]:-$sid}"
    status="fail"
    container_state=""
    if [[ -f "$result_file" ]]; then
        result=$(cat "$result_file")
        # Parse result format: status:sid:container_state
        IFS=':' read -r status sid_check container_state <<< "$result"
    fi
    result_set "$sid" "$status"

    if [[ "$status" == "ok" ]]; then
        log "  ${GREEN}✓${NC} $name - healthy"
    else
        # Core service down → critical, per the documented exit codes
        CRITICAL_FAIL=true
        ANY_FAIL=true
        # Use container state for better error message
        case "$container_state" in
            not_found)
                log "  ${YELLOW}!${NC} $name - container not found"
                ;;
            exited|stopped)
                log "  ${YELLOW}!${NC} $name - container stopped"
                ;;
            restarting)
                log "  ${YELLOW}!${NC} $name - container restarting"
                ;;
            running)
                log "  ${YELLOW}!${NC} $name - not responding (container running)"
                ;;
            *)
                log "  ${YELLOW}!${NC} $name - not responding"
                ;;
        esac
    fi
done

log ""
log "${CYAN}Extension Services:${NC}"

# Launch all enabled extension services in parallel. Disabled extensions
# (no active compose fragment — same predicate as sr_list_enabled) are not
# part of this install's stack and must not be probed or degrade the status.
declare -a EXT_PIDS=()
declare -a EXT_SIDS=()
for sid in "${SERVICE_IDS[@]}"; do
    [[ "${SERVICE_CATEGORIES[$sid]}" == "core" ]] && continue
    ext_compose="${SERVICE_COMPOSE[$sid]:-}"
    [[ -n "$ext_compose" && -f "$ext_compose" ]] || continue
    result_file="$TEMP_DIR/ext_$sid"
    check_service_async "$sid" "$result_file" &
    EXT_PIDS+=($!)
    EXT_SIDS+=("$sid")
done

# Wait for all extension service checks to complete
for pid in "${EXT_PIDS[@]}"; do
    wait "$pid" 2>/dev/null || true
done

# Display extension service results — same aggregation as core services,
# but a down extension only degrades the stack instead of marking it critical.
for sid in "${EXT_SIDS[@]}"; do
    result_file="$TEMP_DIR/ext_$sid"
    name="${SERVICE_NAMES[$sid]:-$sid}"
    status="fail"
    container_state=""
    if [[ -f "$result_file" ]]; then
        result=$(cat "$result_file")
        # Parse result format: status:sid:container_state
        IFS=':' read -r status sid_check container_state <<< "$result"
    fi
    result_set "$sid" "$status"

    if [[ "$status" == "ok" ]]; then
        log "  ${GREEN}✓${NC} $name - healthy"
    else
        ANY_FAIL=true
        # Use container state for better error message
        case "$container_state" in
            not_found)
                log "  ${YELLOW}!${NC} $name - container not found"
                ;;
            exited|stopped)
                log "  ${YELLOW}!${NC} $name - container stopped"
                ;;
            restarting)
                log "  ${YELLOW}!${NC} $name - container restarting"
                ;;
            running)
                log "  ${YELLOW}!${NC} $name - not responding (container running)"
                ;;
            *)
                log "  ${YELLOW}!${NC} $name - not responding"
                ;;
        esac
    fi
done

log ""
log "${CYAN}System Resources:${NC}"

# GPU
if test_gpu 2>/dev/null; then
    status_icon="${GREEN}✓${NC}"
    [ "$(result_get "gpu")" = "warn" ] && status_icon="${YELLOW}!${NC}"
    log "  ${status_icon} GPU - $(result_get "gpu_mem_used")/$(result_get "gpu_mem_total") MiB, $(result_get "gpu_util")% util, $(result_get "gpu_temp")°C"
else
    log "  ${YELLOW}?${NC} GPU - status unavailable"
fi

# Disk
if test_disk 2>/dev/null; then
    status_icon="${GREEN}✓${NC}"
    [ "$(result_get "disk")" = "warn" ] && status_icon="${YELLOW}!${NC}"
    log "  ${status_icon} Disk - $(result_get "disk_usage")% used"
else
    log "  ${YELLOW}?${NC} Disk - status unavailable"
fi

log ""

# Summary
if $CRITICAL_FAIL; then
    log "${RED}Status: CRITICAL - Core services down${NC}"
    EXIT_CODE=2
elif $ANY_FAIL; then
    log "${YELLOW}Status: DEGRADED - Some services unavailable${NC}"
    EXIT_CODE=1
else
    log "${GREEN}Status: HEALTHY - All services operational${NC}"
    EXIT_CODE=0
fi

log ""

# JSON output
if $JSON_OUTPUT; then
    echo "{"
    echo "  \"timestamp\": \"$(date -u +"%Y-%m-%dT%H:%M:%SZ")\","
    echo "  \"status\": \"$([ $EXIT_CODE -eq 0 ] && echo "healthy" || ([ $EXIT_CODE -eq 1 ] && echo "degraded" || echo "critical"))\","
    echo "  \"services\": {"
    first=true
    for i in "${!RESULT_KEYS[@]}"; do
        $first || echo ","
        first=false
        echo -n "    \"${RESULT_KEYS[$i]}\": \"${RESULT_VALS[$i]}\""
    done
    echo ""
    echo "  }"
    echo "}"
fi

exit $EXIT_CODE