Skip to content

armgit / ODS

Public Imported from GitHub
0

ODS V3 Pre-Release: Public testing and refinement ahead of the official V3 launch. Turn your PC, Mac, or Linux box into a private AI server.

28.1 KB
Raw Blame
#!/bin/bash
#=============================================================================
# ods-test.sh - ODS Validation Suite (M8)
#
# Mission M8: Agent Bench Testing Systems
# Validates all critical paths: LLM, STT, TTS, embeddings, tool calling,
# voice round-trip. Clear pass/fail with actionable errors.
#
# Target: <2 minutes runtime
# Pass criteria: All critical tests pass
#
# Usage:
#   ./ods-test.sh                  # Run all tests
#   ./ods-test.sh --quick          # Fast mode (~30s, no inference)
#   ./ods-test.sh --json           # JSON output for automation
#   ./ods-test.sh --service llm     # Test specific service
#
# Exit codes:
#   0 - All critical tests passed
#   1 - One or more tests failed
#   2 - Configuration error
#=============================================================================

set -euo pipefail

# Configuration
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# The install this script ships in: its .env sits beside scripts/.
ODS_DIR="${ODS_DIR:-$(cd "$SCRIPT_DIR/.." && pwd)}"
ENV_FILE="${ENV_FILE:-$ODS_DIR/.env}"
TIMEOUT=15
QUICK_TIMEOUT=5

# Source service registry for port resolution
_DT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
if [[ -f "$_DT_DIR/lib/service-registry.sh" ]]; then
    export SCRIPT_DIR="$_DT_DIR"
    . "$_DT_DIR/lib/service-registry.sh"
    sr_load
    [[ -f "$_DT_DIR/lib/safe-env.sh" ]] && . "$_DT_DIR/lib/safe-env.sh"
    load_env_file "$ENV_FILE"
    sr_resolve_ports
fi

# Service endpoints — resolved from registry
LLM_HOST="${LLM_HOST:-localhost}"
LLM_PORT="${LLM_PORT:-${SERVICE_PORTS[llama-server]:-11434}}"
LLM_URL="http://${LLM_HOST}:${LLM_PORT}"
LLM_HEALTH_PATH="/health"
LLM_ENGINE="llama-server"
LLM_GATEWAY_KEY=""
# LLM_BACKEND=vllm: vllm-server publishes no host port. ODS clients reach it
# through LiteLLM as ods/current (model-router adapts the request to vLLM), so
# the LLM checks go there too, with the gateway key (_llm_chat).
_ods_test_backend="$(printf '%s' "${LLM_BACKEND:-}" | tr '[:upper:]' '[:lower:]')"
if [[ "$_ods_test_backend" == "vllm" ]]; then
    LLM_URL="http://${LLM_HOST}:${LITELLM_PORT:-4000}"
    LLM_HEALTH_PATH="/health/readiness"
    LLM_ENGINE="vllm-server"
    LLM_GATEWAY_KEY="${LITELLM_KEY:-${LITELLM_MASTER_KEY:-}}"
fi
unset _ods_test_backend
WHISPER_HOST="${WHISPER_HOST:-localhost}"
WHISPER_PORT="${WHISPER_PORT:-${SERVICE_PORTS[whisper]:-9000}}"
TTS_HOST="${TTS_HOST:-localhost}"
TTS_PORT="${TTS_PORT:-${SERVICE_PORTS[tts]:-8880}}"
EMBEDDING_HOST="${EMBEDDING_HOST:-localhost}"
EMBEDDING_PORT="${EMBEDDING_PORT:-${SERVICE_PORTS[embeddings]:-9103}}"
LIVEKIT_HOST="${LIVEKIT_HOST:-localhost}"
LIVEKIT_PORT="${LIVEKIT_PORT:-7880}"
PRIVACY_SHIELD_PORT="${PRIVACY_SHIELD_PORT:-${SERVICE_PORTS[privacy-shield]:-8085}}"

# Colors (ANSI escape sequences)
RED='\e[0;31m'
GREEN='\e[0;32m'
YELLOW='\e[1;33m'
BLUE='\e[0;34m'
CYAN='\e[0;36m'
NC='\e[0m'
BOLD='\e[1m'

# Test tracking
TOTAL_TESTS=0
PASSED_TESTS=0
FAILED_TESTS=0
SKIPPED_TESTS=0
START_TIME=0

# Mode flags
JSON_OUTPUT=false
QUICK_MODE=false
SPECIFIC_SERVICE=""
VERBOSE=false

# Results storage
RESULTS_STATUS=()
RESULTS_NAMES=()
RESULTS_DETAILS=()

#--------------------------------------------------------------------------
# Utility Functions
#--------------------------------------------------------------------------

load_env() {
    [[ -f "$_DT_DIR/lib/safe-env.sh" ]] && . "$_DT_DIR/lib/safe-env.sh"
    load_env_file "$ENV_FILE"
}

log() {
    [[ "$VERBOSE" == "true" ]] && echo "$@" >&2
    # Always succeed: log is the last command of `[[ ... ]] && log` guards,
    # and a non-zero status there aborts the run under set -e.
    return 0
}

# Portable millisecond timestamp (macOS BSD date lacks %N)
_now_ms() {
    python3 -c 'import time; print(int(time.time() * 1000))' 2>/dev/null || echo "$(date +%s)000"
}

# The model id llama-server serves (its --alias, the GGUF file name), read from
# /v1/models as test_llm always has; "local" when the listing fails, and the
# request that follows then reports the fault.
_llm_model_id() {
    # The gateway serves the active model under its stable alias.
    if [[ -n "$LLM_GATEWAY_KEY" ]]; then
        echo "ods/current"
        return 0
    fi
    local model_id
    model_id=$(curl -s --max-time 10 "$LLM_URL/v1/models" 2>/dev/null | grep -o '"id":"[^"]*"' | head -1 | cut -d'"' -f4) || model_id=""
    echo "${model_id:-local}"
}

# POST one chat completion ($2) to the LLM route within $1 seconds. A gateway
# key travels on curl's stdin (-H @-), never in argv.
_llm_chat() {
    local seconds="$1" payload="$2"
    if [[ -n "$LLM_GATEWAY_KEY" ]]; then
        printf 'Authorization: Bearer %s\n' "$LLM_GATEWAY_KEY" \
            | curl -s --max-time "$seconds" -X POST "$LLM_URL/v1/chat/completions" \
                -H "Content-Type: application/json" -H @- \
                -d "$payload" 2>/dev/null
    else
        curl -s --max-time "$seconds" -X POST "$LLM_URL/v1/chat/completions" \
            -H "Content-Type: application/json" \
            -d "$payload" 2>/dev/null
    fi
}

# Judge a chat completion (read on stdin) by its JSON structure, not by
# grepping for a word: a reply with "tool_calls": null and no call contains the
# word too. $1 is "text" (an assistant answer; a thinking model may spend a
# short budget on reasoning alone) or "tool" (a get_weather call with a city).
_completion_ok() {
    python3 -c '
import json, sys

def has_weather_call(message):
    for call in message.get("tool_calls") or []:
        function = call.get("function") or {}
        arguments = function.get("arguments")
        if isinstance(arguments, str):
            arguments = json.loads(arguments)
        if (function.get("name") == "get_weather" and isinstance(arguments, dict)
                and str(arguments.get("city") or "").strip()):
            return True
    return False

try:
    message = json.load(sys.stdin)["choices"][0]["message"]
    if sys.argv[1] == "tool":
        ok = has_weather_call(message)
    else:
        text = message.get("content") or message.get("reasoning_content")
        ok = isinstance(text, str) and bool(text.strip())
except (ValueError, LookupError, TypeError, AttributeError):
    ok = False  # not a chat completion: an error body, HTML, or nothing
sys.exit(0 if ok else 1)
' "$1"
}

# Section banner; silent in --json mode so stdout stays one JSON document.
print_section() {
    if [[ "$JSON_OUTPUT" != "true" ]]; then
        echo ""
        echo "> $1"
    fi
}

print_header() {
    if [[ "$JSON_OUTPUT" != "true" ]]; then
        echo ""
        echo "========================================"
        echo "  ODS - VALIDATION SUITE"
        echo "  Mission M8: Critical Path Testing"
        echo "========================================"
        echo ""
    fi
}

record_result() {
    local name="$1"
    local status="$2"
    local details="${3:-}"
    
    RESULTS_NAMES+=("$name")
    RESULTS_STATUS+=("$status")
    RESULTS_DETAILS+=("$details")
    TOTAL_TESTS=$((TOTAL_TESTS + 1))
    
    case "$status" in
        pass) PASSED_TESTS=$((PASSED_TESTS + 1)) ;;
        fail) FAILED_TESTS=$((FAILED_TESTS + 1)) ;;
        skip) SKIPPED_TESTS=$((SKIPPED_TESTS + 1)) ;;
    esac
}

print_test() {
    local name="$1"
    local status="$2"
    local details="${3:-}"
    
    if [[ "$JSON_OUTPUT" != "true" ]]; then
        local icon
        case "$status" in
            pass) icon="OK" ;;
            fail) icon="FAIL" ;;
            skip) icon="SKIP" ;;
        esac
        
        printf "  %-40s [%s]" "$name" "$icon"
        [[ -n "$details" ]] && printf " %s" "$details"
        echo ""
    fi
}

#--------------------------------------------------------------------------
# Core Test Functions
#--------------------------------------------------------------------------

test_http() {
    local name="$1"
    local url="$2"
    local expected="${3:-200}"
    local method="${4:-GET}"
    local payload="${5:-}"
    local custom_timeout="${6:-$TIMEOUT}"
    
    local response_code
    local start_time end_time duration_ms
    
    start_time=$(_now_ms)

    if [[ -n "$payload" && "$method" == "POST" ]]; then
        # curl prints "000" itself when no response arrives, so a fallback in
        # the substitution would append a second "000" ("HTTP 000000").
        response_code=$(curl -s -o /dev/null -w "%{http_code}" \
            --max-time "$custom_timeout" \
            -X POST "$url" \
            -H "Content-Type: application/json" \
            -d "$payload" 2>/dev/null) || response_code="000"
    else
        response_code=$(curl -s -o /dev/null -w "%{http_code}" \
            --max-time "$custom_timeout" \
            "$url" 2>/dev/null) || response_code="000"
    fi
    
    end_time=$(_now_ms)
    duration_ms=$(( end_time - start_time ))

    if [[ "$response_code" == "$expected" ]]; then
        record_result "$name" "pass" "${duration_ms}ms"
        print_test "$name" "pass" "${duration_ms}ms"
        return 0
    else
        record_result "$name" "fail" "HTTP $response_code"
        print_test "$name" "fail" "HTTP $response_code"
        return 1
    fi
}

test_tcp() {
    local name="$1"
    local host="$2"
    local port="$3"
    
    if timeout "$TIMEOUT" bash -c "cat < /dev/null > /dev/tcp/$host/$port" 2>/dev/null; then
        record_result "$name" "pass"
        print_test "$name" "pass"
        return 0
    else
        record_result "$name" "fail" "port $port unreachable"
        print_test "$name" "fail" "port closed"
        return 1
    fi
}

#--------------------------------------------------------------------------
# Service Test Suites
#--------------------------------------------------------------------------

test_docker() {
    print_section "Docker Infrastructure"
    
    if ! command -v docker &>/dev/null; then
        record_result "Docker Available" "fail" "docker not installed"
        print_test "Docker Available" "fail" "not installed"
        return 0
    fi
    
    record_result "Docker Available" "pass"
    print_test "Docker Available" "pass"
    
    if ! timeout 10 docker info &>/dev/null; then
        record_result "Docker Daemon" "fail" "daemon not running"
        print_test "Docker Daemon" "fail" "not running"
        return 0
    fi
    
    record_result "Docker Daemon" "pass"
    print_test "Docker Daemon" "pass"
    
    local running_count
    if ! running_count=$(timeout 10 docker ps --format '{{.Names}}' 2>/dev/null | wc -l); then
        record_result "Running Containers" "fail" "container listing failed"
        print_test "Running Containers" "fail" "container listing failed"
        return 0
    fi
    record_result "Running Containers" "pass" "$running_count containers"
    print_test "Running Containers" "pass" "$running_count containers"
    return 0  # results are tallied by record_result; never surface a status to set -e
}

test_gpu() {
    print_section "GPU Resources"
    
    if ! command -v nvidia-smi &>/dev/null; then
        record_result "NVIDIA GPU" "skip" "nvidia-smi not found"
        print_test "NVIDIA GPU" "skip"
        return 0
    fi
    
    local gpu_info
    gpu_info=$(timeout 10 nvidia-smi --query-gpu=name,memory.used,memory.total,utilization.gpu --format=csv,noheader 2>/dev/null | head -1) || gpu_info=""
    
    if [[ -n "$gpu_info" ]]; then
        local name mem_used mem_total util mem_pct
        name=$(echo "$gpu_info" | cut -d',' -f1 | xargs)
        mem_used=$(echo "$gpu_info" | cut -d',' -f2 | xargs | cut -d' ' -f1)
        mem_total=$(echo "$gpu_info" | cut -d',' -f3 | xargs | cut -d' ' -f1)
        
        if [[ ! "$mem_used" =~ ^[0-9]+$ || ! "$mem_total" =~ ^[0-9]+$ ]] || [[ "$mem_total" -eq 0 ]]; then
            record_result "GPU Memory" "fail" "invalid memory telemetry"
            print_test "GPU Memory" "fail" "invalid memory telemetry"
            return 0
        fi
        mem_pct=$(( mem_used * 100 / mem_total ))
        
        record_result "GPU Available" "pass" "$name"
        print_test "GPU Available" "pass" "$name"
        
        if [[ $mem_pct -gt 90 ]]; then
            record_result "GPU Memory" "fail" "${mem_pct}% used - critical"
            print_test "GPU Memory" "fail" "${mem_pct}% used"
        else
            record_result "GPU Memory" "pass" "${mem_pct}% used"
            print_test "GPU Memory" "pass" "${mem_pct}% used"
        fi
    else
        record_result "NVIDIA GPU" "fail" "no GPU detected"
        print_test "NVIDIA GPU" "fail" "not detected"
    fi
    return 0  # results are tallied by record_result; never surface a status to set -e
}

test_llm() {
    print_section "LLM Inference ($LLM_ENGINE)"

    # A failed check is already recorded; return 0 so the short-circuit only
    # skips this section's dependent checks instead of aborting the whole run
    # under set -e (which lost the summary and the --json document).
    test_http "LLM Health" "$LLM_URL$LLM_HEALTH_PATH" "200" || return 0
    # The gateway lists only its aliases, and only with its key.
    if [[ -z "$LLM_GATEWAY_KEY" ]]; then
        test_http "LLM Models API" "$LLM_URL/v1/models" "200" || log "LLM models API check failed"
    fi

    if [[ "$QUICK_MODE" == "true" ]]; then
        record_result "LLM Inference" "skip" "quick mode"
        print_test "LLM Inference" "skip"
        return 0
    fi

    local model_id
    model_id=$(_llm_model_id)

    local payload="{\"model\": \"$model_id\", \"messages\": [{\"role\": \"user\", \"content\": \"Say hello\"}], \"max_tokens\": 10}"
    local response

    response=$(_llm_chat 30 "$payload") || response=""

    if printf '%s' "$response" | _completion_ok text; then
        local tokens_used
        tokens_used=$(echo "$response" | grep -o '"total_tokens":[0-9]*' | cut -d: -f2) || tokens_used="unknown"
        record_result "LLM Inference" "pass" "${tokens_used} tokens"
        print_test "LLM Inference" "pass" "${tokens_used} tokens"
    else
        record_result "LLM Inference" "fail" "no content in response"
        print_test "LLM Inference" "fail"
        return 0
    fi
    return 0  # results are tallied by record_result; never surface a status to set -e
}

test_tool_calling() {
    print_section "Tool Calling M8 Critical"
    
    if [[ "$QUICK_MODE" == "true" ]]; then
        record_result "Tool Calling" "skip" "quick mode"
        print_test "Tool Calling" "skip"
        return 0
    fi
    
    local model_id
    model_id=$(_llm_model_id)
    local tools='[{"type":"function","function":{"name":"get_weather","description":"Get weather","parameters":{"type":"object","properties":{"city":{"type":"string"}},"required":["city"]}}}]'
    local payload="{\"model\": \"$model_id\", \"messages\": [{\"role\": \"user\", \"content\": \"What is the weather in Tokyo?\"}], \"tools\": $tools, \"tool_choice\": \"auto\", \"max_tokens\": 100}"
    
    local response
    response=$(_llm_chat 30 "$payload") || response=""

    if printf '%s' "$response" | _completion_ok tool; then
        record_result "Tool Calling" "pass" "function called"
        print_test "Tool Calling" "pass" "function called"
    else
        record_result "Tool Calling" "fail" "no get_weather call in the reply"
        print_test "Tool Calling" "fail" "no tool call"
    fi
    return 0  # results are tallied by record_result; never surface a status to set -e
}

test_whisper() {
    print_section "Whisper Speech-to-Text"
    
    test_tcp "Whisper Port" "$WHISPER_HOST" "$WHISPER_PORT" || log "Whisper port check failed"
    
    local health_url="http://${WHISPER_HOST}:${WHISPER_PORT}/health"
    local response
    response=$(curl -s --max-time "$TIMEOUT" "$health_url" 2>/dev/null || echo "")
    
    if [[ -n "$response" ]]; then
        if echo "$response" | grep -qiE "ok|healthy|ready"; then
            record_result "Whisper Health" "pass"
            print_test "Whisper Health" "pass"
        else
            record_result "Whisper Health" "pass" "responding"
            print_test "Whisper Health" "pass" "responding"
        fi
    else
        whisper_http_exit=0
        test_http "Whisper HTTP" "http://${WHISPER_HOST}:${WHISPER_PORT}/" "200" || whisper_http_exit=$?
        [[ $whisper_http_exit -ne 0 ]] && log "Whisper HTTP check failed (exit $whisper_http_exit)"
    fi
    return 0  # results are tallied by record_result; never surface a status to set -e
}

test_tts() {
    print_section "Kokoro TTS Text-to-Speech"
    
    test_tcp "TTS Port" "$TTS_HOST" "$TTS_PORT" || log "TTS port check failed"
    
    local voices_url="http://${TTS_HOST}:${TTS_PORT}/v1/audio/voices"
    local response
    response=$(curl -s --max-time "$TIMEOUT" "$voices_url" 2>/dev/null || echo "")
    
    if echo "$response" | grep -q '"voices"'; then
        local voice_count
        voice_count=$(echo "$response" | grep -o '"voice_id"' | wc -l) || voice_count=0
        record_result "TTS Voices" "pass" "$voice_count voices"
        print_test "TTS Voices" "pass" "$voice_count voices"
    else
        tts_api_exit=0
        test_http "TTS API" "http://${TTS_HOST}:${TTS_PORT}/" "200" || tts_api_exit=$?
        [[ $tts_api_exit -ne 0 ]] && log "TTS API check failed (exit $tts_api_exit)"
    fi
    return 0  # results are tallied by record_result; never surface a status to set -e
}

test_embeddings() {
    print_section "Embeddings TEI"
    
    test_tcp "Embeddings Port" "$EMBEDDING_HOST" "$EMBEDDING_PORT" || log "Embeddings port check failed"
    
    local health_url="http://${EMBEDDING_HOST}:${EMBEDDING_PORT}/health"
    local response
    response=$(curl -s --max-time "$TIMEOUT" "$health_url" 2>/dev/null || echo "")
    
    if echo "$response" | grep -q "ok"; then
        record_result "Embeddings Health" "pass"
        print_test "Embeddings Health" "pass"
    else
        local payload='{"inputs": "test sentence"}'
        embeddings_api_exit=0
        test_http "Embeddings API" "http://${EMBEDDING_HOST}:${EMBEDDING_PORT}/embed" "200" "POST" "$payload" || embeddings_api_exit=$?
        [[ $embeddings_api_exit -ne 0 ]] && log "Embeddings API check failed (exit $embeddings_api_exit)"
    fi
    return 0  # results are tallied by record_result; never surface a status to set -e
}

test_voice_roundtrip() {
    print_section "Voice Round-Trip M8 Critical"
    
    if [[ "$QUICK_MODE" == "true" ]]; then
        record_result "Voice Round-Trip" "skip" "quick mode"
        print_test "Voice Round-Trip" "skip"
        return 0
    fi
    
    local whisper_ready=false
    local tts_ready=false
    local llm_ready=false
    
    if curl -s --max-time 5 "http://${WHISPER_HOST}:${WHISPER_PORT}/health" &>/dev/null; then
        whisper_ready=true
    fi
    if curl -s --max-time 5 "http://${TTS_HOST}:${TTS_PORT}/v1/audio/voices" &>/dev/null; then
        tts_ready=true
    fi
    if curl -s --max-time 5 "$LLM_URL$LLM_HEALTH_PATH" &>/dev/null; then
        llm_ready=true
    fi
    
    if [[ "$whisper_ready" != "true" ]]; then
        record_result "Voice Round-Trip" "skip" "Whisper unavailable"
        print_test "Voice Round-Trip" "skip" "Whisper unavailable"
        return 0
    fi
    if [[ "$tts_ready" != "true" ]]; then
        record_result "Voice Round-Trip" "skip" "TTS unavailable"
        print_test "Voice Round-Trip" "skip" "TTS unavailable"
        return 0
    fi
    if [[ "$llm_ready" != "true" ]]; then
        record_result "Voice Round-Trip" "skip" "LLM unavailable"
        print_test "Voice Round-Trip" "skip" "LLM unavailable"
        return 0
    fi
    
    local start_time end_time duration_ms
    start_time=$(_now_ms)

    local model_id
    model_id=$(_llm_model_id)
    local llm_payload="{\"model\": \"$model_id\", \"messages\": [{\"role\": \"user\", \"content\": \"What is the weather today?\"}], \"max_tokens\": 50}"
    local llm_response
    llm_response=$(_llm_chat 15 "$llm_payload") || llm_response=""
    
    if ! printf '%s' "$llm_response" | _completion_ok text; then
        record_result "Voice Round-Trip" "fail" "LLM step failed"
        print_test "Voice Round-Trip" "fail" "LLM failed"
        return 0
    fi
    
    local tts_text="The weather today is sunny and 75 degrees."
    local tts_payload="{\"model\": \"kokoro\", \"input\": \"$tts_text\", \"voice\": \"af_bella\"}"
    local tts_response
    tts_response=$(curl -s --max-time 15 \
        -X POST "http://${TTS_HOST}:${TTS_PORT}/v1/audio/speech" \
        -H "Content-Type: application/json" \
        -d "$tts_payload" 2>/dev/null) || tts_response=""
    
    end_time=$(_now_ms)
    duration_ms=$(( end_time - start_time ))

    if [[ -n "$tts_response" ]] && [[ ${#tts_response} -gt 100 ]]; then
        record_result "Voice Round-Trip" "pass" "${duration_ms}ms"
        print_test "Voice Round-Trip" "pass" "${duration_ms}ms"
    else
        record_result "Voice Round-Trip" "fail" "TTS step failed"
        print_test "Voice Round-Trip" "fail" "TTS failed"
    fi
    return 0  # results are tallied by record_result; never surface a status to set -e
}

test_privacy_shield() {
    print_section "Privacy Shield M3"
    
    local shield_url="http://localhost:${PRIVACY_SHIELD_PORT}"
    
    test_http "Privacy Shield Health" "$shield_url/health" "200" || return 0
    
    if [[ "$QUICK_MODE" == "true" ]]; then
        record_result "Privacy Shield Proxy" "skip" "quick mode"
        print_test "Privacy Shield Proxy" "skip"
        return 0
    fi
    
    local payload='{"model": "test", "messages": [{"role": "user", "content": "My email is [email protected]"}], "max_tokens": 10}'
    local response
    response=$(curl -s --max-time 10 \
        -X POST "$shield_url/v1/chat/completions" \
        -H "Content-Type: application/json" \
        -d "$payload" 2>/dev/null || echo "")
    
    if [[ -n "$response" ]]; then
        record_result "Privacy Shield Proxy" "pass" "PII scrubbing active"
        print_test "Privacy Shield Proxy" "pass" "PII scrubbing"
    else
        record_result "Privacy Shield Proxy" "fail" "no response"
        print_test "Privacy Shield Proxy" "fail"
    fi
    return 0  # results are tallied by record_result; never surface a status to set -e
}

test_livekit() {
    print_section "LiveKit Voice Infrastructure"
    
    test_tcp "LiveKit Port" "$LIVEKIT_HOST" "$LIVEKIT_PORT" || log "LiveKit port check failed"
    livekit_health_exit=0
    test_http "LiveKit Health" "http://${LIVEKIT_HOST}:${LIVEKIT_PORT}/" "200" || livekit_health_exit=$?
    [[ $livekit_health_exit -ne 0 ]] && log "LiveKit health check failed (exit $livekit_health_exit)"
    return 0  # results are tallied by record_result; never surface a status to set -e
}

#--------------------------------------------------------------------------
# Summary and Output
#--------------------------------------------------------------------------

print_summary() {
    local end_time elapsed_secs
    end_time=$(date +%s)
    elapsed_secs=$((end_time - START_TIME))
    
    if [[ "$JSON_OUTPUT" == "true" ]]; then
        _print_json_summary "$elapsed_secs"
    else
        _print_text_summary "$elapsed_secs"
    fi
}

_print_json_summary() {
    local elapsed="$1"
    
    echo "{"
    echo "  \"timestamp\": \"$(date -u +"%Y-%m-%dT%H:%M:%SZ")\","
    echo "  \"runtime_seconds\": $elapsed,"
    echo "  \"summary\": {"
    echo "    \"total\": $TOTAL_TESTS,"
    echo "    \"passed\": $PASSED_TESTS,"
    echo "    \"failed\": $FAILED_TESTS,"
    echo "    \"skipped\": $SKIPPED_TESTS,"
    echo "    \"success\": $([ $FAILED_TESTS -eq 0 ] && echo 'true' || echo 'false')"
    echo "  },"
    echo "  \"results\": ["
    
    local first=true
    local i
    for i in "${!RESULTS_NAMES[@]}"; do
        [[ "$first" == "true" ]] || echo ","
        printf '    {"name": "%s", "status": "%s", "details": "%s"}' \
            "${RESULTS_NAMES[$i]}" "${RESULTS_STATUS[$i]}" "${RESULTS_DETAILS[$i]:-}"
        first=false
    done
    echo ""
    echo "  ]"
    echo "}"
}

_print_text_summary() {
    local elapsed="$1"
    
    echo ""
    echo "========================================"
    echo ""
    
    if [[ $FAILED_TESTS -eq 0 ]]; then
        echo -e "  ${GREEN}ALL TESTS PASSED${NC}"
        echo ""
        echo "  Passed: $PASSED_TESTS | Skipped: $SKIPPED_TESTS"
        echo "  Runtime: ${elapsed}s"
        echo ""
        echo -e "  ${BOLD}ODS is ready!${NC}"
    else
        echo -e "  ${RED}SOME TESTS FAILED${NC}"
        echo ""
        echo -e "  Passed: ${GREEN}${PASSED_TESTS}${NC} | Failed: ${RED}${FAILED_TESTS}${NC} | Skipped: ${YELLOW}${SKIPPED_TESTS}${NC}"
        echo "  Runtime: ${elapsed}s"
        echo ""
        echo "  Failed tests:"
        local i
        for i in "${!RESULTS_NAMES[@]}"; do
            if [[ "${RESULTS_STATUS[$i]}" == "fail" ]]; then
                echo "    - ${RESULTS_NAMES[$i]}"
            fi
        done
        echo ""
        echo "Actionable fixes:"
        
        if [[ "${RESULTS_STATUS[0]:-}" == "fail" ]] && [[ "${RESULTS_NAMES[0]:-}" == *"LLM"* ]]; then
            echo "  - LLM not responding - check: docker logs ods-${LLM_ENGINE}"
        fi

        local i
        for i in "${!RESULTS_NAMES[@]}"; do
            if [[ "${RESULTS_STATUS[$i]}" == "fail" ]]; then
                case "${RESULTS_NAMES[$i]}" in
                    "Tool Calling") echo "  - Tool calling failed - check $LLM_ENGINE tool support" ;;
                    "Whisper Port") echo "  - Whisper not running - start: docker compose up whisper" ;;
                    "TTS Port") echo "  - TTS not running - start: docker compose up kokoro-tts" ;;
                esac
            fi
        done
        echo "  Need help? Ask on the ODS Discord: https://discord.gg/4ntNp9MAwC"
    fi
    
    echo ""
}

#--------------------------------------------------------------------------
# Main
#--------------------------------------------------------------------------

show_help() {
    cat << 'EOF'
ODS Validation Suite (M8)

USAGE:
    ods-test.sh [OPTIONS]

OPTIONS:
    --quick, -q        Fast mode (~30s, no inference tests)
    --json, -j         Output results as JSON
    --service, -s      Test specific service only
    --verbose, -v      Show detailed output
    --help, -h         Show this help

SERVICES:
    docker, gpu, llm, tool-calling, whisper, tts,
    embeddings, voice-roundtrip, privacy-shield, livekit

EXAMPLES:
    ods-test.sh                    # Run all tests
    ods-test.sh --quick            # Fast health check
    ods-test.sh --json > results.json
    ods-test.sh --service llm      # Test LLM only

EXIT CODES:
    0 - All tests passed
    1 - One or more tests failed
    2 - Configuration error

EOF
}

run_all_tests() {
    print_header
    
    test_docker
    test_gpu
    test_llm
    test_tool_calling
    test_whisper
    test_tts
    test_embeddings
    test_voice_roundtrip
    test_privacy_shield
    test_livekit
}

run_specific_service() {
    local service="$1"
    
    print_header
    
    case "$service" in
        docker)          test_docker ;;
        gpu)             test_gpu ;;
        llm)             test_llm ;;
        tool-calling)    test_tool_calling ;;
        whisper)         test_whisper ;;
        tts)             test_tts ;;
        embeddings)      test_embeddings ;;
        voice-roundtrip) test_voice_roundtrip ;;
        privacy-shield)  test_privacy_shield ;;
        livekit)         test_livekit ;;
        *)
            echo "Unknown service: $service" >&2
            echo "Available: docker, gpu, llm, tool-calling, whisper, tts, embeddings, voice-roundtrip, privacy-shield, livekit" >&2
            exit 2
            ;;
    esac
}

main() {
    START_TIME=$(date +%s)
    
    # Parse arguments
    while [[ $# -gt 0 ]]; do
        case "$1" in
            --json|-j)
                JSON_OUTPUT=true
                shift
                ;;
            --quick|-q)
                QUICK_MODE=true
                TIMEOUT=$QUICK_TIMEOUT
                shift
                ;;
            --service|-s)
                SPECIFIC_SERVICE="$2"
                shift 2
                ;;
            --verbose|-v)
                VERBOSE=true
                shift
                ;;
            --help|-h)
                show_help
                exit 0
                ;;
            *)
                echo "Unknown option: $1" >&2
                show_help
                exit 2
                ;;
        esac
    done
    
    # Load environment
    load_env
    
    # Run tests
    if [[ -n "$SPECIFIC_SERVICE" ]]; then
        run_specific_service "$SPECIFIC_SERVICE"
    else
        run_all_tests
    fi
    
    # Print summary
    print_summary
    
    # Exit with appropriate code
    if [[ $FAILED_TESTS -eq 0 ]]; then
        exit 0
    else
        exit 1
    fi
}

main "$@"