ODS V3 Pre-Release: Public testing and refinement ahead of the official V3 launch. Turn your PC, Mac, or Linux box into a private AI server.
Python ai-agentsamdcomfyuidockerllama-cppllmlocal-ain8nnvidiaopen-webuiragself-hostedspeech-to-textstrix-halotext-to-speechworkflow-automation
#!/usr/bin/env bash
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ROOT_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"
usage() {
echo "Usage: $0 [REPORT_PATH]"
echo " $0 --help"
echo ""
echo "Generates a machine-readable diagnostics report for installer and runtime readiness."
echo "Report includes capability profile, preflight-style analysis, and autofix_hints."
echo ""
echo "Arguments:"
echo " REPORT_PATH Output JSON path (default: /tmp/ods-doctor-report.json)"
echo ""
echo "Exit codes: 0 = report generated, 1 = error (e.g. missing dependency)"
echo ""
echo "See docs/ODS-DOCTOR.md for details."
}
case "${1:-}" in
-h|--help) usage; exit 0 ;;
esac
# ods-doctor sources lib/service-registry.sh, which uses Bash 4 associative
# arrays. macOS ships Bash 3.2, and every caller launches this script by its
# shebang (ods-cli's `ods doctor`, installers/macos.sh, the dashboard host
# agent, and the documented `scripts/ods-doctor.sh` invocation), so it runs
# under whatever `bash` is first on PATH — often /bin/bash 3.2. Without a
# guard the report failed to generate with "service-registry.sh requires
# Bash 4.0+". Re-exec under a modern Bash, mirroring install-macos.sh.
_ods_doctor_bash_is_modern() {
[ -x "$1" ] || return 1
# The test string must evaluate in the candidate shell, not this one.
# shellcheck disable=SC2016
"$1" -c '[ "${BASH_VERSINFO[0]:-0}" -ge 4 ]' >/dev/null 2>&1
}
if [ "${BASH_VERSINFO[0]:-0}" -lt 4 ]; then
_ods_doctor_bash_candidates=("${BASH:-}" /opt/homebrew/bin/bash /usr/local/bin/bash)
if command -v brew >/dev/null 2>&1; then
_ods_doctor_brew_prefix="$(brew --prefix 2>/dev/null)"
[ -n "$_ods_doctor_brew_prefix" ] && _ods_doctor_bash_candidates=("$_ods_doctor_brew_prefix/bin/bash" "${_ods_doctor_bash_candidates[@]}")
fi
for _ods_doctor_candidate in "${_ods_doctor_bash_candidates[@]}"; do
if _ods_doctor_bash_is_modern "$_ods_doctor_candidate"; then
exec "$_ods_doctor_candidate" "$0" "$@"
fi
done
echo "ods-doctor.sh requires Bash 4+ (you have ${BASH_VERSION})." >&2
echo "macOS ships only Bash 3.2; install a modern one with: brew install bash" >&2
exit 1
fi
REPORT_FILE="${1:-/tmp/ods-doctor-report.json}"
# Scratch files live in a private directory for this run only. Fixed names in
# the shared /tmp let an earlier run by another user (sudo ods doctor) make
# every later run fail before it wrote its report.
DOCTOR_WORK_DIR="$(mktemp -d "${TMPDIR:-/tmp}/ods-doctor.XXXXXX")"
trap 'rm -rf -- "$DOCTOR_WORK_DIR"' EXIT
CAP_FILE="$DOCTOR_WORK_DIR/capabilities.json"
PREFLIGHT_FILE="$DOCTOR_WORK_DIR/preflight.json"
DOCTOR_BASH_CMD="${BASH:-}"
if [[ -z "$DOCTOR_BASH_CMD" || ! -x "$DOCTOR_BASH_CMD" ]]; then
DOCTOR_BASH_CMD="$(command -v bash 2>/dev/null || printf '%s\n' bash)"
fi
# Source service registry and safe env helpers
if [[ -f "$ROOT_DIR/lib/service-registry.sh" ]]; then
export SCRIPT_DIR="$ROOT_DIR"
. "$ROOT_DIR/lib/service-registry.sh"
sr_load
fi
if [[ -f "$ROOT_DIR/lib/safe-env.sh" ]]; then
. "$ROOT_DIR/lib/safe-env.sh"
fi
# Safe .env loading (no direct source to avoid injection). Prefer the shared
# reader from lib/safe-env.sh, sourced above: it applies Compose's grammar
# (quotes, inline comments after a quoted or unquoted value) and skips the
# readonly UID that older .env files still carry. The loop below is only the
# fallback for a tree without lib/.
load_env_safe() {
local env_file="${1:-$ROOT_DIR/.env}"
[[ -f "$env_file" ]] || return 0
if declare -F load_env_file >/dev/null 2>&1; then
load_env_file "$env_file"
return 0
fi
while IFS='=' read -r key value; do
value="${value%$'\r'}"
[[ "$key" =~ ^[[:space:]]*# ]] && continue
[[ -z "$key" ]] && continue
[[ "$key" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]] || continue
value="${value%\"}"
value="${value#\"}"
value="${value%\'}"
value="${value#\'}"
export "$key=$value"
done < "$env_file"
}
load_env_safe "$ROOT_DIR/.env"
sr_resolve_ports
_DASHBOARD_PORT="${SERVICE_PORTS[dashboard]:-3001}"
_WEBUI_PORT="${SERVICE_PORTS[open-webui]:-3000}"
# RAM: platform-branch. /proc/meminfo does not exist on macOS; use sysctl.
if [[ "$(uname -s)" == "Darwin" ]]; then
RAM_BYTES="$(sysctl -n hw.memsize 2>/dev/null || echo 0)"
RAM_GB=$(( RAM_BYTES / 1024 / 1024 / 1024 ))
else
RAM_GB="$(grep MemTotal /proc/meminfo 2>/dev/null | awk '{print int($2/1024/1024)}' || echo 0)"
fi
# Installer-recorded fallback: if detection returned 0 and .env has HOST_RAM_GB, trust that.
if (( RAM_GB == 0 )) && [[ -f "$ROOT_DIR/.env" ]]; then
_env_ram=$(grep '^HOST_RAM_GB=' "$ROOT_DIR/.env" | cut -d= -f2 | tr -d '"' || true)
[[ -n "${_env_ram:-}" ]] && RAM_GB="$_env_ram"
fi
_doctor_disk_free_gb() {
local path="$1" value
value="$(df -k "$path" 2>/dev/null | tail -1 | awk '{print int($4/1024/1024)}' || true)"
[[ "$value" =~ ^[0-9]+$ ]] || value=0
printf '%s\n' "$value"
}
_doctor_external_inference_enabled() {
# A host-native llama-server (Windows Portal) keeps its model on Windows.
[[ -n "${EXTERNAL_LLM_URL:-}" \
|| -n "${NATIVE_LLM_BASE_URL:-}" \
|| "${ODS_MODE:-local}" == "cloud" ]]
}
_doctor_select_disk() {
local home_path="${HOME:-$ROOT_DIR}" docker_root="" docker_disk_gb=0
DOCTOR_HOME_DISK_GB="$(_doctor_disk_free_gb "$home_path")"
DOCTOR_DISK_SOURCE="$home_path"
DISK_GB="$DOCTOR_HOME_DISK_GB"
# External inference stores neither the served model nor Docker images
# under HOME. When Docker has a separate data-root, apply the existing
# tier floor to the filesystem that actually grows during installation.
# Fall back to HOME if the daemon or data-root cannot be observed.
if _doctor_external_inference_enabled && command -v docker >/dev/null 2>&1; then
docker_root="$(docker info --format '{{.DockerRootDir}}' 2>/dev/null || true)"
if [[ "$docker_root" == /* ]]; then
docker_disk_gb="$(_doctor_disk_free_gb "$docker_root")"
if [[ "$docker_disk_gb" =~ ^[0-9]+$ && "$docker_disk_gb" -gt 0 ]]; then
DISK_GB="$docker_disk_gb"
DOCTOR_DISK_SOURCE="$docker_root"
fi
fi
fi
export DISK_GB DOCTOR_HOME_DISK_GB DOCTOR_DISK_SOURCE
}
# POSIX df -k works on BSD and GNU identically (df -BG is GNU-only).
_doctor_select_disk
if [[ -x "$SCRIPT_DIR/scripts/build-capability-profile.sh" ]]; then
CAP_ENV="$("$DOCTOR_BASH_CMD" "$SCRIPT_DIR/scripts/build-capability-profile.sh" --output "$CAP_FILE" --env)"
load_env_from_output <<< "$CAP_ENV"
else
echo "scripts/build-capability-profile.sh not found/executable" >&2
exit 1
fi
if [[ -x "$SCRIPT_DIR/scripts/preflight-engine.sh" ]]; then
PREFLIGHT_ENV="$("$DOCTOR_BASH_CMD" "$SCRIPT_DIR/scripts/preflight-engine.sh" \
--report "$PREFLIGHT_FILE" \
--tier "${CAP_RECOMMENDED_TIER:-T1}" \
--ram-gb "$RAM_GB" \
--disk-gb "$DISK_GB" \
--disk-policy runtime \
--gpu-backend "${CAP_LLM_BACKEND:-cpu}" \
--gpu-vram-mb "${CAP_GPU_VRAM_MB:-0}" \
--gpu-name "${CAP_GPU_NAME:-Unknown}" \
--platform-id "${CAP_PLATFORM_ID:-unknown}" \
--compose-overlays "${CAP_COMPOSE_OVERLAYS:-}" \
--script-dir "$ROOT_DIR" \
--env)"
load_env_from_output <<< "$PREFLIGHT_ENV"
else
echo "scripts/preflight-engine.sh not found/executable" >&2
exit 1
fi
DOCKER_CLI="false"
DOCKER_DAEMON="false"
# Exported for the report writer below, keeping its argument list unchanged.
export DOCKER_ACCESS_DENIED="false"
COMPOSE_CLI="false"
DASHBOARD_HTTP="false"
WEBUI_HTTP="false"
# Open WebUI is optional. ENABLE_OPEN_WEBUI is disabled only by the exact
# case-sensitive literal "false"; unset or any other value keeps it enabled.
# Keep this consistent with ods-cli's optional-service status check.
WEBUI_ENABLED="true"
if [[ "${ENABLE_OPEN_WEBUI:-}" == "false" ]]; then
WEBUI_ENABLED="false"
fi
# Extension diagnostics (JSON array of objects)
EXT_DIAGNOSTICS="[]"
if command -v docker >/dev/null 2>&1; then
DOCKER_CLI="true"
if docker_info_error="$(docker info 2>&1 >/dev/null)"; then
DOCKER_DAEMON="true"
elif [[ "$docker_info_error" == *"permission denied"* ]]; then
# A running daemon this account cannot use yet, as right after the
# installer adds it to the docker group, is not a stopped daemon.
DOCKER_ACCESS_DENIED="true"
fi
if docker compose version >/dev/null 2>&1 || command -v docker-compose >/dev/null 2>&1; then
COMPOSE_CLI="true"
fi
fi
if command -v curl >/dev/null 2>&1; then
if curl -sf --max-time 10 "http://127.0.0.1:${_DASHBOARD_PORT}" >/dev/null 2>&1; then
DASHBOARD_HTTP="true"
fi
if [[ "$WEBUI_ENABLED" == "true" ]] \
&& curl -sf --max-time 10 "http://127.0.0.1:${_WEBUI_PORT}" >/dev/null 2>&1; then
WEBUI_HTTP="true"
fi
fi
# Colors for doctor logging
if [[ -t 1 ]] && [[ -z "${NO_COLOR+x}" ]]; then
_GREEN=$'\033[0;32m'
_RED=$'\033[0;31m'
_BLUE=$'\033[0;34m'
_YELLOW=$'\033[1;33m'
_NC=$'\033[0m'
else
_GREEN=''
_RED=''
_BLUE=''
_YELLOW=''
_NC=''
fi
log_ok() { echo -e " ${_GREEN}✓${_NC} $*"; }
log_fail() { echo -e " ${_RED}✗${_NC} $*"; }
log_info() { echo -e " ${_BLUE}ℹ${_NC} $*"; }
log_warn() { echo -e " ${_YELLOW}⚠${_NC} $*"; }
LLM_STATUS="unknown"
LLM_PROVIDER=""
LLM_MODEL=""
LLM_URL=""
LLM_LOCAL_WARNING="false"
LLM_RECOVERY=""
_doctor_check_external_llm() {
local url="$1" provider="$2" model="$3"
local health_path
local probe_ok=false
LLM_URL="$url"
LLM_PROVIDER="${provider:-external}"
LLM_MODEL="$model"
LLM_RECOVERY=""
LLM_LOCAL_WARNING="false"
case "$provider" in
ollama) health_path="/api/tags" ;;
lmstudio) health_path="/v1/models" ;;
*) health_path="/v1/models" ;; # OpenAI-compat fallback
esac
# API mode passes the key file the installer stored: a keyed API answers
# 401 without it, which is not "down". The key reaches curl as a header
# file, never as an argument. Other callers keep the plain probe.
local key_file="${4:-}" key="" status="000"
LLM_FAILURE=""
if [[ -n "$key_file" && -s "$key_file" && ! -r "$key_file" ]]; then
LLM_FAILURE="key-unreadable"
elif [[ -n "$key_file" ]] && command -v curl >/dev/null 2>&1; then
if [[ -s "$key_file" ]]; then
IFS= read -r key < "$key_file" || true # a key without a final newline still reads
fi
if [[ -n "$key" ]]; then
status="$(curl -s -o /dev/null --max-time 5 -w '%{http_code}' \
-H @<(printf 'Authorization: Bearer %s\n' "$key") "${url%/}${health_path}" 2>/dev/null)" || true
else
status="$(curl -s -o /dev/null --max-time 5 -w '%{http_code}' "${url%/}${health_path}" 2>/dev/null)" || true
fi
case "$status" in
2[0-9][0-9]) probe_ok=true ;;
401|403) if [[ -n "$key" ]]; then LLM_FAILURE="key-refused"; else LLM_FAILURE="key-required"; fi ;;
*) LLM_FAILURE="unreachable" ;;
esac
key=""
elif command -v curl >/dev/null 2>&1 \
&& curl -sf --max-time 5 "${url%/}${health_path}" > /dev/null 2>&1; then
probe_ok=true
fi
if [[ "$LLM_FAILURE" == key-unreadable ]]; then
LLM_STATUS="unknown"
log_warn "LLM backend: ${provider:-external} (external) — not checked: this user cannot read its stored API key"
log_info " Endpoint : $url"
log_info " Recovery : run ods doctor as the user that installed ODS"
elif [[ "$LLM_FAILURE" == key-refused || "$LLM_FAILURE" == key-required ]]; then
LLM_STATUS="fail"
if [[ "$LLM_FAILURE" == key-refused ]]; then
log_fail "LLM backend: ${provider:-external} (external) — the API refused the stored key (HTTP $status)"
LLM_RECOVERY="replace the API key: rerun the installer with --external-llm-key-file FILE (Windows: -ExternalLlmKeyFile FILE)"
else
log_fail "LLM backend: ${provider:-external} (external) — the API needs a key, and none is stored (HTTP $status)"
LLM_RECOVERY="add the API key: rerun the installer with --external-llm-key-file FILE (Windows: -ExternalLlmKeyFile FILE)"
fi
log_info " Endpoint : $url"
log_info " Recovery : $LLM_RECOVERY"
elif [[ "$probe_ok" == true ]]; then
LLM_STATUS="ok"
log_ok "LLM backend: ${provider:-external} (external) — responding"
log_ok " Endpoint : $url"
[ -n "$model" ] && log_ok " Model : $model"
else
LLM_STATUS="fail"
log_fail "LLM backend: ${provider:-external} (external) — not responding"
log_info " Endpoint : $url"
case "$provider" in
ollama)
LLM_RECOVERY="ollama serve"
log_info " Recovery : ollama serve"
;;
lmstudio)
LLM_RECOVERY="start LM Studio and enable local server"
log_info " Recovery : start LM Studio and enable local server"
;;
*)
LLM_RECOVERY="check external LLM service connectivity"
;;
esac
fi
if [[ "$DOCKER_DAEMON" == "true" ]] && docker ps --format '{{.Names}}' 2>/dev/null | grep -qx 'ods-llama-server'; then
LLM_LOCAL_WARNING="true"
log_warn " Note : Unused local llama-server container is running"
fi
}
# The Windows Portal's llama-server runs outside the stack. Its /health needs
# no API key, so the probe sends none.
_doctor_check_host_native_llm() {
local url="${NATIVE_LLM_BASE_URL%/}" reached=false
LLM_URL="$url"
LLM_PROVIDER="llama-server (Windows)"
LLM_MODEL="${GGUF_FILE:-}"
LLM_RECOVERY=""
LLM_LOCAL_WARNING="false"
if [[ "${ODS_HOST_LLM_TRANSPORT:-}" == model-router && -n "${NATIVE_LLM_CONTAINER_BASE_URL:-}" ]]; then
# Under WSL's default NAT networking this shell cannot reach Windows
# loopback; the Portal's route goes through model-router
# (host.docker.internal). Probe from that container, as LiteLLM and
# the router reach the server.
url="${NATIVE_LLM_CONTAINER_BASE_URL%/}"
LLM_URL="$url"
if [[ "${DOCKER_DAEMON:-false}" == "true" ]] && docker exec ods-model-router python -c \
'import sys, urllib.request; urllib.request.urlopen(sys.argv[1] + "/health", timeout=5)' \
"$url" >/dev/null 2>&1; then
reached=true
fi
elif command -v curl >/dev/null 2>&1 && curl -sf --max-time 5 "${url}/health" >/dev/null 2>&1; then
reached=true
fi
if $reached; then
LLM_STATUS="ok"
log_ok "LLM backend: llama-server on Windows (host-native) — responding"
log_ok " Endpoint : $url"
[[ -z "$LLM_MODEL" ]] || log_ok " Model : $LLM_MODEL"
else
LLM_STATUS="fail"
LLM_RECOVERY="restart the llama-server task from the ODS Portal on Windows, then rerun setup there"
if [[ "${ODS_HOST_LLM_TRANSPORT:-}" == model-router ]]; then
LLM_RECOVERY="if ods-model-router is not running, run ods start; otherwise $LLM_RECOVERY"
fi
log_fail "LLM backend: llama-server on Windows (host-native) — not responding"
log_info " Endpoint : $url"
log_info " Recovery : $LLM_RECOVERY"
fi
# Informational only: a failed listing just omits the note.
if [[ "$DOCKER_DAEMON" == "true" ]] && docker ps --format '{{.Names}}' 2>/dev/null | grep -qx 'ods-llama-server'; then
LLM_LOCAL_WARNING="true"
log_warn " Note : Unused local llama-server container is running"
fi
}
_doctor_check_llama_server() {
local port="${OLLAMA_PORT:-${LLAMA_SERVER_PORT:-${SERVICE_PORTS[llama-server]:-11434}}}"
local health_path="${SERVICE_HEALTH[llama-server]:-/health}"
LLM_PROVIDER="llama-server"
LLM_RECOVERY=""
if [[ "$(uname -s)" == Darwin ]]; then
port="${ODS_NATIVE_LLAMA_PORT:-8080}"
# Native inference stays private even when the dashboard uses a LAN bind.
local probe_host="127.0.0.1"
LLM_URL=""
if [[ "$port" =~ ^[0-9]+$ && ${#port} -le 5 ]] \
&& (( 10#$port > 0 && 10#$port <= 65535 )) && [[ -n "$probe_host" ]]; then
LLM_URL="http://${probe_host}:${port}"
fi
if [[ -n "$LLM_URL" ]] && command -v curl >/dev/null 2>&1 \
&& curl -sf --max-time 5 "${LLM_URL}/health" >/dev/null 2>&1; then
LLM_STATUS="ok"
log_ok "LLM backend: llama-server (native Metal) - responding"
log_ok " Endpoint : $LLM_URL"
else
LLM_STATUS="fail"
LLM_RECOVERY="check the native llama-server process and ODS_NATIVE_LLAMA_PORT; run ods restart"
log_fail "LLM backend: llama-server (native Metal) - not responding"
log_info " Recovery : $LLM_RECOVERY"
fi
return
fi
local container_name
container_name=$(sr_container "llama-server" 2>/dev/null || echo "ods-llama-server")
LLM_URL="http://127.0.0.1:${port}"
LLM_PROVIDER="llama-server"
if [[ "$DOCKER_DAEMON" == "true" ]] && docker ps --format '{{.Names}}' 2>/dev/null | grep -qx "$container_name"; then
if command -v curl >/dev/null 2>&1 && curl -sf --max-time 5 "http://127.0.0.1:${port}${health_path}" >/dev/null 2>&1; then
LLM_STATUS="ok"
log_ok "LLM backend: llama-server (local) — responding"
log_ok " Endpoint : http://127.0.0.1:${port}"
else
LLM_STATUS="fail"
LLM_RECOVERY="ods restart llama-server"
log_fail "LLM backend: llama-server (local) — container running but not responding"
log_info " Endpoint : http://127.0.0.1:${port}"
log_info " Recovery : ods restart llama-server"
fi
else
LLM_STATUS="fail"
LLM_RECOVERY="ods start llama-server"
log_fail "LLM backend: llama-server (local) — container not running"
log_info " Recovery : ods start llama-server"
fi
}
# LLM_BACKEND=vllm (docker-compose.vllm.yml): vllm-server serves the model on
# the ODS network without a host port, and the in-stack llama-server is off.
# Docker's healthcheck asks vLLM's /health, which answers only once the model
# is loaded, so its state is the engine's status. A cold start loads the
# weights and compiles kernels for up to 15 minutes: that is starting, not a
# fault. A llama-server container left by an earlier install still running
# would compete for the GPU.
_doctor_check_vllm_engine() {
local inspect="" state="" health=""
LLM_PROVIDER="vLLM"
LLM_URL="http://vllm-server:8000 (ODS network only)"
LLM_MODEL="${VLLM_SERVED_MODEL:-}"
LLM_RECOVERY=""
LLM_LOCAL_WARNING="false"
if [[ "$DOCKER_DAEMON" == "true" ]] && inspect=$(docker inspect --format \
'{{.State.Status}} {{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' \
ods-vllm-server 2>&1); then
read -r state health <<<"$inspect"
fi
if [[ "$state" == "running" && "$health" == "healthy" ]]; then
LLM_STATUS="ok"
log_ok "LLM backend: vLLM (local) — responding"
[[ -z "$LLM_MODEL" ]] || log_ok " Model : $LLM_MODEL"
elif [[ "$state" == "running" && "$health" == "starting" ]]; then
LLM_STATUS="starting"
log_info "LLM backend: vLLM (local) — starting: a first start loads the model and compiles kernels, which can take 15 minutes"
log_info " Follow : ods logs llm"
else
LLM_STATUS="fail"
if [[ "$state" == "running" ]]; then
LLM_RECOVERY="ods restart vllm-server"
log_fail "LLM backend: vLLM (local) — container running but not healthy"
else
LLM_RECOVERY="ods start vllm-server"
log_fail "LLM backend: vLLM (local) — container not running"
fi
log_info " Recovery : $LLM_RECOVERY (see why with: ods logs llm)"
fi
if [[ "$DOCKER_DAEMON" == "true" ]] && docker ps --format '{{.Names}}' 2>/dev/null | grep -qx 'ods-llama-server'; then
LLM_LOCAL_WARNING="true"
log_warn " Note : A llama-server container from an earlier install is running beside vLLM; stop it with: docker stop ods-llama-server"
fi
}
_doctor_check_llm_backend() {
local ext_url="${EXTERNAL_LLM_URL:-}"
local ext_provider="${EXTERNAL_LLM_PROVIDER:-}"
local ext_model="${EXTERNAL_LLM_MODEL:-}"
local mode="${ODS_MODE:-local}"
local backend
backend=$(printf '%s' "${LLM_BACKEND:-}" | tr '[:upper:]' '[:lower:]')
if [ -n "$ext_url" ]; then
# External LLM mode — skip llama-server check
_doctor_check_external_llm "$ext_url" "$ext_provider" "$ext_model" \
"$ROOT_DIR/config/litellm/external-upstream.key"
elif [[ -n "${NATIVE_LLM_BASE_URL:-}" ]]; then
_doctor_check_host_native_llm
elif [[ "$backend" == "vllm" && "$mode" != "cloud" ]]; then
_doctor_check_vllm_engine
elif [[ "$mode" == "cloud" ]]; then
local cloud_url="${LLM_API_URL:-}"
if [ -n "$cloud_url" ]; then
# Translate known LiteLLM/container routes to the host-mapped port
# E.g. http://litellm:4000 -> http://127.0.0.1:${LITELLM_PORT:-4000}
local probe_url="$cloud_url"
if [[ "$probe_url" =~ ^https?://litellm(:[0-9]+)?(.*)$ ]]; then
local port_part="${BASH_REMATCH[1]:-}"
local path_part="${BASH_REMATCH[2]:-}"
local port="4000"
if [ -n "$port_part" ]; then
port="${port_part#:}"
elif [ -n "${LITELLM_PORT:-}" ]; then
port="$LITELLM_PORT"
fi
probe_url="http://127.0.0.1:${port}${path_part}"
fi
# Extract host to check if it's host-probeable (e.g. contains '.' or is 'localhost')
local host
host=$(echo "$probe_url" | grep -oE '://[^/]+' | cut -d/ -f3 | cut -d: -f1)
local is_probeable=false
if [[ "$host" == "localhost" ]] || [[ "$host" == *"."* ]]; then
is_probeable=true
fi
if [ "$is_probeable" = true ]; then
_doctor_check_external_llm "$probe_url" "${ext_provider:-openai}" "$ext_model"
else
LLM_URL="$cloud_url"
LLM_PROVIDER="cloud"
LLM_STATUS="ok"
LLM_MODEL=""
LLM_RECOVERY=""
log_ok "LLM backend: cloud (external) — container-internal endpoint bypassed host probe"
log_ok " Endpoint : $cloud_url"
fi
else
# Leave backend check provider-neutral when no probe target exists
LLM_URL=""
LLM_PROVIDER="cloud"
LLM_STATUS="ok"
LLM_MODEL=""
LLM_RECOVERY=""
fi
else
# Managed local/hybrid runtimes still use the local container check.
_doctor_check_llama_server
fi
}
_doctor_check_llm_backend
export LLM_STATUS
export LLM_PROVIDER
export LLM_MODEL
export LLM_URL
export LLM_LOCAL_WARNING
export LLM_RECOVERY
export LLM_FAILURE
# STT model cache check: a common silent-failure mode is the installer's
# pre-download failing, so Whisper's /health passes (service up) but the
# model isn't cached. Transcription then returns 404. This check catches
# that case and surfaces the exact recovery command.
STT_MODEL_CACHED="unknown"
STT_MODEL_NAME=""
STT_RECOVERY_HINT=""
TTS_HTTP="unknown"
TTS_PORT=""
if [[ "${ENABLE_VOICE:-false}" == "true" ]] && command -v curl >/dev/null 2>&1; then
STT_MODEL_NAME="${AUDIO_STT_MODEL:-Systran/faster-whisper-base}"
_stt_whisper_port="${SERVICE_PORTS[whisper]:-9000}"
_stt_model_encoded="${STT_MODEL_NAME//\//%2F}"
_stt_whisper_url="http://127.0.0.1:${_stt_whisper_port}"
if curl -sf --max-time 5 "${_stt_whisper_url}/v1/models/${_stt_model_encoded}" >/dev/null 2>&1; then
STT_MODEL_CACHED="true"
else
# Distinguish "service down" from "model missing" for the hint.
if curl -sf --max-time 5 "${_stt_whisper_url}/health" >/dev/null 2>&1; then
STT_MODEL_CACHED="false"
STT_RECOVERY_HINT="curl --max-time 3600 -X POST ${_stt_whisper_url}/v1/models/${_stt_model_encoded}"
else
STT_MODEL_CACHED="service_down"
fi
fi
TTS_PORT="${SERVICE_PORTS[tts]:-8880}"
if curl -sf --max-time 5 "http://127.0.0.1:${TTS_PORT}/health" >/dev/null 2>&1; then
TTS_HTTP="true"
else
TTS_HTTP="false"
fi
elif [[ "${ENABLE_VOICE:-false}" != "true" ]]; then
STT_MODEL_CACHED="disabled"
TTS_HTTP="disabled"
fi
# Compute capability 12.1 CUDA arch check: DGX Spark (GB10) and RTX Spark
# (N1X). llama.cpp's CUDA 12 images (server-cuda-*) load on these GPUs without
# sm_121 kernels and answer with unusable output ("????"); the installer picks
# the CUDA 13 image config/backends/nvidia.json pins for them. Surface a
# mismatch in doctor so operators do not have to infer it from llama-server
# logs. The DGX_SPARK_* names and report keys are kept for older readers.
DGX_SPARK_GPU="false"
DGX_SPARK_GPU_NAME=""
DGX_SPARK_COMPUTE_CAP=""
LLAMA_CUDA_ARCHS=""
DGX_SPARK_CUDA_ARCH_STATUS="unknown"
DGX_SPARK_CUDA_ARCH_MESSAGE=""
_doctor_gpu_backend="${GPU_BACKEND:-${CAP_LLM_BACKEND:-}}"
if [[ "$_doctor_gpu_backend" == "nvidia" ]] && command -v nvidia-smi >/dev/null 2>&1; then
_dgx_gpu_raw="$(nvidia-smi --query-gpu=name,compute_cap --format=csv,noheader,nounits 2>/dev/null | head -1 || true)"
if [[ -n "$_dgx_gpu_raw" ]]; then
DGX_SPARK_GPU_NAME="$(echo "$_dgx_gpu_raw" | cut -d',' -f1 | xargs)"
DGX_SPARK_COMPUTE_CAP="$(echo "$_dgx_gpu_raw" | cut -d',' -f2 | xargs)"
if [[ "$DGX_SPARK_GPU_NAME" == *"GB10"* || "$DGX_SPARK_COMPUTE_CAP" == "12.1" ]]; then
DGX_SPARK_GPU="true"
if [[ "$DOCKER_DAEMON" == "true" ]] && docker ps -a --format '{{.Names}}' 2>/dev/null | grep -qx 'ods-llama-server'; then
_llama_arch_line="$(docker logs ods-llama-server 2>&1 | grep 'CUDA : ARCHS =' | tail -1 || true)"
LLAMA_CUDA_ARCHS="$(echo "$_llama_arch_line" | sed -n 's/.*CUDA : ARCHS = \([^|]*\).*/\1/p' | xargs)"
if [[ -z "$LLAMA_CUDA_ARCHS" ]]; then
DGX_SPARK_CUDA_ARCH_STATUS="unknown"
DGX_SPARK_CUDA_ARCH_MESSAGE="DGX Spark / RTX Spark GPU (compute capability 12.1) detected, but llama-server CUDA archs were not found in logs."
elif [[ ",${LLAMA_CUDA_ARCHS}," == *",1210,"* || ",${LLAMA_CUDA_ARCHS}," == *",121,"* || ",${LLAMA_CUDA_ARCHS}," == *",121a,"* ]]; then
DGX_SPARK_CUDA_ARCH_STATUS="pass"
DGX_SPARK_CUDA_ARCH_MESSAGE="llama-server includes sm_121 kernels for this DGX Spark / RTX Spark GPU."
else
DGX_SPARK_CUDA_ARCH_STATUS="warn"
DGX_SPARK_CUDA_ARCH_MESSAGE="DGX Spark / RTX Spark GPU (compute capability 12.1) detected, but llama-server reports CUDA archs '${LLAMA_CUDA_ARCHS}' without sm_121."
fi
else
DGX_SPARK_CUDA_ARCH_STATUS="unknown"
DGX_SPARK_CUDA_ARCH_MESSAGE="DGX Spark / RTX Spark GPU (compute capability 12.1) detected, but ods-llama-server is not available for CUDA arch inspection."
fi
fi
fi
fi
# Collect extension diagnostics (wrapped in function to allow local variables)
collect_extension_diagnostics() {
# Use outer GPU_BACKEND or default to nvidia (don't make local to avoid set -u issues)
local backend="${GPU_BACKEND-nvidia}"
local EXT_DIAG_ITEMS=()
for sid in "${SERVICE_IDS[@]}"; do
# Skip core services
[[ "${SERVICE_CATEGORIES[$sid]:-}" == "core" ]] && continue
# Check if extension is enabled
local compose_file="${SERVICE_COMPOSE[$sid]:-}"
[[ -z "$compose_file" || ! -f "$compose_file" ]] && continue
# Build diagnostic entry
local container="${SERVICE_CONTAINERS[$sid]:-}"
local container_state="unknown"
local health_status="unknown"
local issues=()
# Check container state
if [[ "$DOCKER_DAEMON" == "true" && -n "$container" ]]; then
local inspect_output
if inspect_output=$(docker inspect --format '{{.State.Status}}' "$container" 2>/dev/null); then
container_state="$inspect_output"
else
container_state="not_found"
fi
# Check health endpoint if container running
if [[ "$container_state" == "running" ]]; then
local port="${SERVICE_PORTS[$sid]:-0}"
local health="${SERVICE_HEALTH[$sid]:-}"
if [[ "$port" != "0" && -n "$health" ]]; then
if curl -sf --max-time 5 "http://127.0.0.1:${port}${health}" >/dev/null 2>&1; then
health_status="healthy"
else
health_status="unhealthy"
issues+=("health_check_failed")
fi
fi
elif [[ "$container_state" == exited \
&& "${SERVICE_PORTS[$sid]:-0}" == 0 \
&& "${SERVICE_STARTUP_CHECKS[$sid]:-true}" == false \
&& "${SERVICE_SOCKET_ONLY[$sid]:-0}" != 1 ]] \
&& [[ "$(docker inspect --format '{{.State.ExitCode}} {{.State.OOMKilled}}' "$container" 2>/dev/null)" == '0 false' ]] \
&& jq -e --arg sid "$sid" 'type == "object" and .service_id == $sid and .status == "started" and .exit_verified == true' \
"$ROOT_DIR/data/extension-progress/$sid.json" >/dev/null 2>&1; then
# CLI tools finish normally; a stopped daemon is still a fault.
health_status="completed"
else
issues+=("container_not_running")
fi
fi
# Check GPU backend compatibility (only if SERVICE_GPU_BACKENDS array exists from PR #357).
# dashboard-api uses GPU_BACKEND=nvidia internally on macOS (see
# installers/macos/docker-compose.macos.yml) so service manifests are
# discovered. doctor/preflight path doesn't have that workaround, so the
# raw gpu_backends check produces false positives for CPU-only services
# declaring gpu_backends: [amd, nvidia]. Skip the check on apple — if a
# service genuinely needs GPU and isn't available on Apple, it's a
# manifest-level concern, not a runtime doctor warning.
if [[ "$backend" != "apple" ]] && declare -p SERVICE_GPU_BACKENDS &>/dev/null; then
local gpu_backends="${SERVICE_GPU_BACKENDS[$sid]:-}"
if [[ -n "$gpu_backends" \
&& ! " $gpu_backends " =~ " all " \
&& ! " $gpu_backends " =~ " $backend " ]]; then
issues+=("gpu_backend_incompatible")
fi
fi
# Check dependencies
local deps="${SERVICE_DEPENDS[$sid]:-}"
if [[ -n "$deps" ]]; then
local dep
for dep in $deps; do
local dep_compose="${SERVICE_COMPOSE[$dep]:-}"
local dep_cat="${SERVICE_CATEGORIES[$dep]:-}"
if [[ "$dep_cat" != "core" && ! -f "$dep_compose" ]]; then
issues+=("missing_dependency:$dep")
fi
done
fi
# Build JSON object (escape quotes in values)
local issues_json="[]"
if [[ ${#issues[@]} -gt 0 ]]; then
# Use printf with newline separator, then convert to JSON array
issues_json="[\"$(printf '%s\n' "${issues[@]}" | sed 's/"/\\"/g' | tr '\n' ',' | sed 's/,$//' | sed 's/,/","/g')\"]"
fi
EXT_DIAG_ITEMS+=("{\"id\":\"$sid\",\"container_state\":\"$container_state\",\"health_status\":\"$health_status\",\"issues\":$issues_json}")
done
if [[ ${#EXT_DIAG_ITEMS[@]} -gt 0 ]]; then
echo "[$(IFS=,; echo "${EXT_DIAG_ITEMS[*]}")]"
else
echo "[]"
fi
}
# Hermes slash workers are spawned by upstream Hermes for /slash commands. A
# leak in that worker lifecycle can create many long-lived child processes and
# starve local models of RAM. Doctor only reports; cleanup remains explicit via
# `ods repair hermes-workers`.
HERMES_SLASH_WORKER_MAX_COUNT="${HERMES_SLASH_WORKER_MAX_COUNT:-8}"
HERMES_SLASH_WORKER_COUNT="0"
if [[ "$DOCKER_DAEMON" == "true" ]] && docker ps --format '{{.Names}}' 2>/dev/null | grep -qx 'ods-hermes'; then
_hermes_worker_count="$(
docker exec ods-hermes sh -c \
"(ps -eo args= 2>/dev/null || ps -ef 2>/dev/null) | grep '[t]ui_gateway[.]slash_worker' | wc -l" \
2>/dev/null || echo 0
)"
HERMES_SLASH_WORKER_COUNT="$(echo "$_hermes_worker_count" | tr -dc '0-9')"
[[ -n "$HERMES_SLASH_WORKER_COUNT" ]] || HERMES_SLASH_WORKER_COUNT="0"
fi
ODS_MANAGED_CONTAINER_COUNT="0"
ODS_RUNNING_CONTAINER_COUNT="0"
if [[ "$DOCKER_DAEMON" == "true" ]]; then
ODS_MANAGED_CONTAINER_COUNT="$(
docker ps -a --format '{{.Names}}' 2>/dev/null | grep -c '^ods-' || true
)"
ODS_RUNNING_CONTAINER_COUNT="$(
docker ps --format '{{.Names}}' 2>/dev/null | grep -c '^ods-' || true
)"
fi
# Rootless Docker subordinate ID ranges — runtime half of the install
# preflight's ROOTLESS_SUBID check. A rootless daemon maps in-container
# users through /etc/subuid + /etc/subgid; a missing entry or a range under
# 65536 IDs breaks layer extraction for non-root images with cryptic
# "uid/gid map out of range" errors, so doctor surfaces it with the fix.
ROOTLESS_DOCKER="false"
ROOTLESS_SUBID_STATUS="skipped"
ROOTLESS_SUBUID_TOTAL="0"
ROOTLESS_SUBGID_TOTAL="0"
ROOTLESS_SUBID_USER="$(id -un 2>/dev/null || echo unknown)"
if [[ "${ODS_ASSUME_ROOTLESS:-0}" == "1" ]] || { [[ "$DOCKER_DAEMON" == "true" ]] \
&& docker info --format '{{.SecurityOptions}}' 2>/dev/null | grep -q 'rootless'; }; then
ROOTLESS_DOCKER="true"
subid_range_total() {
# Sum every range allocated to the user; entries may be keyed by
# name or numeric UID (subgid is also keyed by user, not group).
local file="$1" user="$2" numeric_uid="$3" total=0 owner start count
[[ -r "$file" ]] || { echo 0; return; }
while IFS=: read -r owner start count; do
if [[ "$owner" == "$user" || "$owner" == "$numeric_uid" ]]; then
[[ "$count" =~ ^[0-9]+$ ]] && total=$((total + count))
fi
done <"$file"
echo "$total"
}
_subid_uid="$(id -u)"
ROOTLESS_SUBUID_TOTAL="$(subid_range_total "${ODS_SUBUID_FILE:-/etc/subuid}" "$ROOTLESS_SUBID_USER" "$_subid_uid")"
ROOTLESS_SUBGID_TOTAL="$(subid_range_total "${ODS_SUBGID_FILE:-/etc/subgid}" "$ROOTLESS_SUBID_USER" "$_subid_uid")"
if [[ "$ROOTLESS_SUBUID_TOTAL" -eq 0 || "$ROOTLESS_SUBGID_TOTAL" -eq 0 ]]; then
ROOTLESS_SUBID_STATUS="fail"
elif [[ "$ROOTLESS_SUBUID_TOTAL" -lt 65536 || "$ROOTLESS_SUBGID_TOTAL" -lt 65536 ]]; then
ROOTLESS_SUBID_STATUS="warn"
else
ROOTLESS_SUBID_STATUS="pass"
fi
fi
export ROOTLESS_DOCKER ROOTLESS_SUBID_STATUS ROOTLESS_SUBUID_TOTAL ROOTLESS_SUBGID_TOTAL ROOTLESS_SUBID_USER
# Collect extension diagnostics if service registry loaded
EXT_DIAGNOSTICS="[]"
if [[ "${#SERVICE_IDS[@]}" -gt 0 ]]; then
EXT_DIAGNOSTICS=$(collect_extension_diagnostics)
fi
PYTHON_CMD="python3"
if [[ -f "$ROOT_DIR/lib/python-cmd.sh" ]]; then
. "$ROOT_DIR/lib/python-cmd.sh"
PYTHON_CMD="$(ods_detect_python_cmd)"
elif command -v python >/dev/null 2>&1; then
PYTHON_CMD="python"
fi
"$PYTHON_CMD" - "$CAP_FILE" "$PREFLIGHT_FILE" "$REPORT_FILE" "$DOCKER_CLI" "$DOCKER_DAEMON" "$COMPOSE_CLI" "$DASHBOARD_HTTP" "$WEBUI_HTTP" "$_DASHBOARD_PORT" "$_WEBUI_PORT" "$EXT_DIAGNOSTICS" "$STT_MODEL_CACHED" "$STT_MODEL_NAME" "$STT_RECOVERY_HINT" "$TTS_HTTP" "$TTS_PORT" "$DGX_SPARK_GPU" "$DGX_SPARK_GPU_NAME" "$DGX_SPARK_COMPUTE_CAP" "$LLAMA_CUDA_ARCHS" "$DGX_SPARK_CUDA_ARCH_STATUS" "$DGX_SPARK_CUDA_ARCH_MESSAGE" "$HERMES_SLASH_WORKER_COUNT" "$HERMES_SLASH_WORKER_MAX_COUNT" "$ODS_MANAGED_CONTAINER_COUNT" "$ODS_RUNNING_CONTAINER_COUNT" "$ROOT_DIR" "$WEBUI_ENABLED" <<'PY'
import json
import os
import pathlib
import re
import shlex
import sys
from datetime import datetime, timezone
from urllib import error, parse, request
cap_file, preflight_file, report_file, docker_cli, docker_daemon, compose_cli, dashboard_http, webui_http, dashboard_port, webui_port, ext_diagnostics_json, stt_cached, stt_model_name, stt_recovery, tts_http, tts_port, dgx_spark_gpu, dgx_spark_gpu_name, dgx_spark_compute_cap, llama_cuda_archs, dgx_spark_arch_status, dgx_spark_arch_message, hermes_slash_worker_count, hermes_slash_worker_max_count, ods_managed_container_count, ods_running_container_count, root_dir_arg, webui_enabled = sys.argv[1:]
# Passed in the environment so callers of this block keep its argument list.
docker_access_denied = os.environ.get("DOCKER_ACCESS_DENIED", "false")
cap = json.load(open(cap_file, "r", encoding="utf-8"))
pre = json.load(open(preflight_file, "r", encoding="utf-8"))
ext_diagnostics = json.loads(ext_diagnostics_json)
root_dir = pathlib.Path(root_dir_arg).resolve()
def _int_value(raw, default=0):
try:
return int(str(raw).strip())
except (TypeError, ValueError):
return default
hermes_slash_worker_count_num = _int_value(hermes_slash_worker_count)
hermes_slash_worker_max_count_num = max(1, _int_value(hermes_slash_worker_max_count, 8))
def _clean_env(name, default=""):
return os.environ.get(name, default).strip()
def _join_url(base_url, path):
base = base_url.rstrip("/")
suffix = path if path.startswith("/") else f"/{path}"
return f"{base}{suffix}"
def _split_backends(raw):
backends = []
invalid = []
for item in (raw or "").split(","):
backend = item.strip().lower()
if not backend:
continue
if backend in {"auto", "cpu", "npu", "rocm", "vulkan"}:
if backend not in backends:
backends.append(backend)
else:
invalid.append(backend)
return backends, invalid
def _env_bool(name):
return _clean_env(name).lower() in {"1", "true", "yes", "on"}
def _amd_health_url(location, port):
"""llama-server's /health: the Windows Portal's host-native server, or the container."""
native_base = _clean_env("NATIVE_LLM_BASE_URL").rstrip("/")
if location == "host" and native_base:
return _join_url(native_base, "health")
host_port = _clean_env("OLLAMA_PORT", port) if location == "container" else port
return _join_url(f"http://127.0.0.1:{host_port}", "health")
def _probe_health(url):
try:
with request.urlopen(url, timeout=2.0) as response:
status = getattr(response, "status", response.getcode())
body = response.read(4096).decode("utf-8", errors="replace")
except error.HTTPError as exc:
return "unhealthy", "unknown", f"health_http_{exc.code}"
except (error.URLError, TimeoutError, OSError):
return "unreachable", "unknown", "health_unreachable"
version = "unknown"
try:
payload = json.loads(body) if body else {}
if isinstance(payload, dict) and payload.get("version"):
version = str(payload["version"])
except json.JSONDecodeError:
pass
if 200 <= int(status) < 300:
return "reachable", version, None
return "unhealthy", version, f"health_http_{status}"
def _amd_runtime_report():
gpu_backend = (_clean_env("GPU_BACKEND") or _clean_env("CAP_LLM_BACKEND")).lower()
amd_env_present = any(
_clean_env(name)
for name in (
"AMD_INFERENCE_RUNTIME",
"AMD_INFERENCE_BACKEND",
"AMD_INFERENCE_LOCATION",
"AMD_INFERENCE_SUPPORTED_BACKENDS",
)
)
# In API mode the API serves the model and no AMD runtime runs here, so
# there is nothing to probe (Strixy in API mode reported it unreachable).
api_mode = bool(_clean_env("EXTERNAL_LLM_URL")) or _clean_env("LLM_BACKEND").lower() == "external"
if (gpu_backend != "amd" and not amd_env_present) or api_mode:
return {
"available": False,
"reason": "api_mode" if api_mode else "not_amd",
"runtime": "none",
"location": "none",
"runtimeMode": "none",
"managedByODS": False,
"selectedBackend": "none",
"supportedBackends": [],
"defaultBackend": "none",
"health": "not_checked",
"warnings": [],
}
warnings = []
runtime = _clean_env("AMD_INFERENCE_RUNTIME").lower()
selected_backend = _clean_env("AMD_INFERENCE_BACKEND").lower()
location = _clean_env("AMD_INFERENCE_LOCATION").lower()
runtime_mode = _clean_env("AMD_INFERENCE_RUNTIME_MODE").lower()
supported_backends, invalid_backends = _split_backends(_clean_env("AMD_INFERENCE_SUPPORTED_BACKENDS"))
managed_raw = _clean_env("AMD_INFERENCE_MANAGED").lower()
managed = _env_bool("AMD_INFERENCE_MANAGED")
port = _clean_env("AMD_INFERENCE_PORT", "8080") or "8080"
if invalid_backends:
warnings.append("amd_supported_backends_invalid")
if not runtime:
runtime = "none"
warnings.append("amd_runtime_env_missing")
if not selected_backend:
selected_backend = "unknown"
warnings.append("amd_backend_env_missing")
if not location:
location = "unknown"
warnings.append("amd_location_env_missing")
if not runtime_mode:
runtime_mode = "unknown"
warnings.append("amd_runtime_mode_env_missing")
if not managed_raw:
warnings.append("amd_managed_env_missing")
if not supported_backends:
warnings.append("amd_supported_backends_env_missing")
elif selected_backend not in {"unknown", "none"} and selected_backend not in supported_backends:
warnings.append("amd_selected_backend_not_supported")
if not port.isdigit() or not (1 <= int(port) <= 65535):
port = "8080"
warnings.append("amd_port_invalid")
health = "not_checked"
version = "unknown"
health_url = None
reason = None
if runtime == "llama-server" and location == "host" \
and _clean_env("ODS_HOST_LLM_TRANSPORT") == "model-router" \
and _clean_env("NATIVE_LLM_CONTAINER_BASE_URL"):
# Under WSL's default NAT networking this host cannot reach Windows
# loopback. The LLM backend check above already probed the server
# through ods-model-router, the path the stack uses; report that.
health_url = _join_url(_clean_env("NATIVE_LLM_CONTAINER_BASE_URL").rstrip("/"), "health")
if _clean_env("LLM_STATUS") == "ok":
health = "reachable"
else:
health = "unreachable"
warnings.append("health_unreachable")
elif runtime == "llama-server":
health_url = _amd_health_url(location, port)
health, version, health_warning = _probe_health(health_url)
if health_warning:
warnings.append(health_warning)
elif runtime == "lemonade":
# Compatibility read for one release: an .env the Lemonade migration
# has not rewritten yet. ODS no longer ships that runtime.
reason = "runtime_retired"
warnings.append("amd_runtime_lemonade_retired")
else:
reason = "runtime_not_configured"
return {
"available": runtime == "llama-server",
"reason": reason,
"runtime": runtime,
"location": location,
"runtimeMode": runtime_mode,
"managedByODS": managed,
"selectedBackend": selected_backend,
"supportedBackends": supported_backends,
"defaultBackend": selected_backend if selected_backend else "none",
"healthUrl": health_url,
"health": health,
"version": version,
"warnings": warnings,
}
amd_runtime = _amd_runtime_report()
def _read_text(path, max_chars=120_000):
try:
text = path.read_text(encoding="utf-8", errors="replace")
except OSError:
return ""
if len(text) > max_chars:
return text[-max_chars:]
return text
def _latest_install_report():
reports = []
try:
reports = [p for p in root_dir.glob("install-report-*.txt") if p.is_file()]
except OSError:
reports = []
if not reports:
return None
return max(reports, key=lambda p: p.stat().st_mtime)
def _parse_kv_lines(text):
values = {}
for line in text.splitlines():
if "=" not in line:
continue
key, value = line.split("=", 1)
key = key.strip()
if re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", key):
values[key] = value.strip()
return values
def _truthy(value):
return str(value or "").strip().lower() in {"1", "true", "yes", "on"}
def _int_arg(value):
try:
return int(str(value or "0").strip())
except ValueError:
return 0
def _mtime(path):
try:
return path.stat().st_mtime
except OSError:
return 0
def _source_mentions_zero_containers(text):
lowered = text.lower()
return (
"docker compose did not create any managed containers" in lowered
or "zero managed containers" in lowered
)
def _zero_container_failure_is_stale(failure_sources, compose_launch_path):
if docker_daemon != "true":
return False
if _int_arg(ods_managed_container_count) <= 0 or _int_arg(ods_running_container_count) <= 0:
return False
if not compose_launch_path.exists():
return False
zero_source_mtimes = [
_mtime(path)
for path, text in failure_sources
if path and _source_mentions_zero_containers(text)
]
if not zero_source_mtimes:
return False
return _mtime(compose_launch_path) > max(zero_source_mtimes)
def _env_file_values():
env_path = root_dir / ".env"
if not env_path.exists():
return {}
return _parse_kv_lines(_read_text(env_path))
def _compose_files_from_flags(flags_text):
if not flags_text.strip():
return []
try:
tokens = shlex.split(flags_text)
except ValueError:
tokens = flags_text.split()
files = []
idx = 0
while idx < len(tokens):
token = tokens[idx]
if token in {"-f", "--file"} and idx + 1 < len(tokens):
files.append(tokens[idx + 1])
idx += 2
continue
if token.startswith("--file="):
files.append(token.split("=", 1)[1])
idx += 1
return files
def _has_compose_file(files, expected):
return any(pathlib.PurePosixPath(item).name == expected or item == expected for item in files)
def _looks_like_litellm_route(value):
lowered = (value or "").lower()
return "litellm" in lowered or ":4000" in lowered
def _looks_like_local_llama_route(value):
value = (value or "").strip()
if not value:
return False
try:
route = parse.urlsplit(value if "://" in value or value.startswith("//") else "//" + value)
hostname = (route.hostname or "").lower().rstrip(".")
port = route.port
except ValueError:
return False
# A provider path or query can contain a local URL without selecting it.
# Diagnose the actual destination rather than matching text anywhere.
if hostname in {"llama-server", "ods-llama-server", "model-router", "ods-model-router"}:
return True
return hostname in {"localhost", "127.0.0.1", "::1", "host.docker.internal"} and port in {
8080, 9099, 11434,
}
def _url_hostname(value):
"""The lower-case host of a URL, or "" when it names none."""
try:
return (parse.urlsplit(value.strip()).hostname or "").lower() if "://" in value else ""
except ValueError:
return ""
def _source(path):
try:
return path.relative_to(root_dir).as_posix()
except ValueError:
return path.as_posix()
def _evidence(source, detail):
return {"source": source, "detail": detail}
def _diagnosis(diag_id, severity, confidence, title, evidence, impact, next_steps):
return {
"id": diag_id,
"severity": severity,
"confidence": confidence,
"title": title,
"evidence": evidence,
"impact": impact,
"next_steps": next_steps,
}
def _inference_issue(issue_id, severity, source, detail):
return {
"id": issue_id,
"severity": severity,
"source": source,
"detail": detail,
}
def _collect_inference_contract():
env_values = _env_file_values()
def env_get(name, default=""):
return env_values.get(name) or _clean_env(name, default)
flags_path = root_dir / ".compose-flags"
flags_text = _read_text(flags_path, max_chars=20_000) if flags_path.exists() else ""
compose_flags_exists = flags_path.exists()
compose_files = _compose_files_from_flags(flags_text)
ods_mode = (env_get("ODS_MODE", "local") or "local").strip().lower()
gpu_backend = (env_get("GPU_BACKEND", "") or "").strip().lower()
llm_backend = env_get("LLM_BACKEND", "")
external_llm_url = env_get("EXTERNAL_LLM_URL", "")
llm_api_url = env_get("LLM_API_URL", "")
hermes_base_url = env_get("HERMES_LLM_BASE_URL", "")
# The Windows Portal's llama-server runs outside the stack; LiteLLM and
# model-router reach it with LLAMA_SERVER_API_KEY.
native_llm_url = env_get("NATIVE_LLM_BASE_URL", "")
host_native = bool(native_llm_url.strip())
native_key_configured = bool(env_get("LLAMA_SERVER_API_KEY", "").strip())
# Compatibility read for one release: Lemonade-era settings the migration
# has not rewritten yet (the installer runs it; `ods update` does not).
lemonade_era = (
ods_mode == "lemonade"
or llm_backend.strip().lower() == "lemonade"
or _truthy(env_get("LEMONADE_EXTERNAL"))
or env_get("AMD_INFERENCE_RUNTIME").strip().lower() == "lemonade"
)
cloud_overlay = _has_compose_file(compose_files, "docker-compose.cloud.yml")
host_native_overlay = _has_compose_file(compose_files, "docker-compose.host-native-llm.yml")
local_inference_overlay = any(
_has_compose_file(compose_files, name)
for name in (
"docker-compose.nvidia.yml",
"docker-compose.amd.yml",
"docker-compose.cpu.yml",
"docker-compose.arc.yml",
"docker-compose.intel.yml",
"docker-compose.apple.yml",
"docker-compose.macos.yml",
)
)
generic_external = bool(external_llm_url.strip()) or llm_backend.strip().lower() == "external"
external_inference = ods_mode == "cloud" or generic_external
# LLM_BACKEND=vllm: vllm-server serves the model behind model-router;
# llama-server is off and clients use the LiteLLM gateway.
vllm_engine = llm_backend.strip().lower() == "vllm" and not external_inference
vllm_overlay = _has_compose_file(compose_files, "docker-compose.vllm.yml")
expected_owner = "external" if external_inference else "ods"
expected_gateway = "litellm" if external_inference or host_native or vllm_engine else "llama-server"
issues = []
if lemonade_era:
issues.append(
_inference_issue(
"ODS-RUNTIME-LEMONADE-RETIRED",
"blocker",
".env",
"The .env still selects the Lemonade runtime, which ODS no longer ships.",
)
)
elif ods_mode not in {"local", "cloud", "hybrid"}:
issues.append(
_inference_issue(
"ODS-RUNTIME-MODE-UNKNOWN",
"blocker",
".env",
f"ODS_MODE={ods_mode!r} is not one of local/cloud/hybrid.",
)
)
if ods_mode == "cloud":
if compose_flags_exists and not cloud_overlay:
issues.append(
_inference_issue(
"ODS-RUNTIME-CLOUD-OVERLAY-MISSING",
"blocker",
".compose-flags",
"ODS_MODE=cloud but docker-compose.cloud.yml is not in the resolved compose stack.",
)
)
if _looks_like_local_llama_route(llm_api_url):
issues.append(
_inference_issue(
"ODS-RUNTIME-CLOUD-LLM-LOCAL-ROUTE",
"blocker",
".env",
f"ODS_MODE=cloud but LLM_API_URL points at local llama-server ({llm_api_url}).",
)
)
if _looks_like_local_llama_route(hermes_base_url):
issues.append(
_inference_issue(
"ODS-RUNTIME-CLOUD-HERMES-LOCAL-ROUTE",
"blocker",
".env",
f"ODS_MODE=cloud but HERMES_LLM_BASE_URL points at local llama-server ({hermes_base_url}).",
)
)
if llm_api_url and not _looks_like_litellm_route(llm_api_url):
issues.append(
_inference_issue(
"ODS-RUNTIME-CLOUD-GATEWAY-BYPASS",
"warn",
".env",
f"ODS_MODE=cloud normally routes ODS services through LiteLLM, but LLM_API_URL={llm_api_url}.",
)
)
if host_native:
if compose_flags_exists and cloud_overlay:
issues.append(
_inference_issue(
"ODS-RUNTIME-HOST-NATIVE-CLOUD-OVERLAY-CONFLICT",
"blocker",
".compose-flags",
"The cloud overlay disables model-router; the Windows llama-server route uses its own overlay so Pixel and model switching remain live.",
)
)
if compose_flags_exists and not host_native_overlay:
issues.append(
_inference_issue(
"ODS-RUNTIME-HOST-NATIVE-OVERLAY-MISSING",
"warn",
".compose-flags",
"NATIVE_LLM_BASE_URL is set but docker-compose.host-native-llm.yml is not in the compose stack.",
)
)
if _looks_like_local_llama_route(llm_api_url):
issues.append(
_inference_issue(
"ODS-RUNTIME-HOST-NATIVE-LOCAL-ROUTE",
"blocker",
".env",
f"The model runs in llama-server on Windows, but LLM_API_URL points at the in-stack llama-server ({llm_api_url}).",
)
)
if not native_key_configured:
issues.append(
_inference_issue(
"ODS-RUNTIME-HOST-NATIVE-KEY-MISSING",
"warn",
".env",
"LLAMA_SERVER_API_KEY is empty, so LiteLLM and model-router send no key to the Windows llama-server, which requires one.",
)
)
if vllm_engine:
if compose_flags_exists and not vllm_overlay:
issues.append(
_inference_issue(
"ODS-RUNTIME-VLLM-OVERLAY-MISSING",
"blocker",
".compose-flags",
"LLM_BACKEND=vllm but docker-compose.vllm.yml is not in the resolved compose stack.",
)
)
if _url_hostname(llm_api_url) in {"llama-server", "ods-llama-server"}:
issues.append(
_inference_issue(
"ODS-RUNTIME-VLLM-LLAMA-ROUTE",
"blocker",
".env",
f"The vLLM engine serves the model, but LLM_API_URL points at the in-stack llama-server ({llm_api_url}), which is off.",
)
)
if ods_mode == "local" and not host_native and not generic_external and not lemonade_era and not vllm_engine:
if compose_flags_exists and cloud_overlay:
issues.append(
_inference_issue(
"ODS-RUNTIME-LOCAL-CLOUD-OVERLAY",
"warn",
".compose-flags",
"ODS_MODE=local but docker-compose.cloud.yml is still in the compose stack.",
)
)
if _looks_like_litellm_route(llm_api_url):
issues.append(
_inference_issue(
"ODS-RUNTIME-LOCAL-LITELLM-ROUTE",
"warn",
".env",
f"ODS_MODE=local routes directly to llama-server on every GPU, but LLM_API_URL={llm_api_url}.",
)
)
diagnoses = []
issue_titles = {
"ODS-RUNTIME-MODE-UNKNOWN": "Runtime mode is not recognized",
"ODS-RUNTIME-CLOUD-OVERLAY-MISSING": "Cloud mode is missing the cloud compose overlay",
"ODS-RUNTIME-CLOUD-LLM-LOCAL-ROUTE": "Cloud mode still routes chat clients to local llama-server",
"ODS-RUNTIME-CLOUD-HERMES-LOCAL-ROUTE": "Cloud mode still routes Hermes to local llama-server",
"ODS-RUNTIME-CLOUD-GATEWAY-BYPASS": "Cloud mode bypasses the LiteLLM gateway",
"ODS-RUNTIME-LEMONADE-RETIRED": "Settings still select the retired Lemonade runtime",
"ODS-RUNTIME-HOST-NATIVE-CLOUD-OVERLAY-CONFLICT": "The Windows llama-server route incorrectly includes the cloud compose overlay",
"ODS-RUNTIME-HOST-NATIVE-OVERLAY-MISSING": "The Windows llama-server route is missing its compose overlay",
"ODS-RUNTIME-HOST-NATIVE-LOCAL-ROUTE": "The Windows llama-server route still sends clients to the in-stack llama-server",
"ODS-RUNTIME-HOST-NATIVE-KEY-MISSING": "The Windows llama-server route has no API key",
"ODS-RUNTIME-LOCAL-CLOUD-OVERLAY": "Local mode still has the cloud compose overlay",
"ODS-RUNTIME-LOCAL-LITELLM-ROUTE": "Local mode routes through LiteLLM unexpectedly",
"ODS-RUNTIME-VLLM-OVERLAY-MISSING": "The vLLM route is missing its compose overlay",
"ODS-RUNTIME-VLLM-LLAMA-ROUTE": "The vLLM route still sends clients to the in-stack llama-server",
}
next_steps = {
"ODS-RUNTIME-CLOUD-OVERLAY-MISSING": [
"Regenerate compose flags with ODS_MODE=cloud and include docker-compose.cloud.yml.",
"Run `ods restart` after correcting .env/.compose-flags.",
],
"ODS-RUNTIME-CLOUD-LLM-LOCAL-ROUTE": [
"Set LLM_API_URL to the LiteLLM service URL used by the stack, usually http://litellm:4000.",
"Do not point cloud mode at llama-server unless ODS is intentionally managing local inference.",
],
"ODS-RUNTIME-CLOUD-HERMES-LOCAL-ROUTE": [
"Set HERMES_LLM_BASE_URL to http://litellm:4000/v1 for cloud mode.",
"Restart Hermes after updating the generated config/template.",
],
"ODS-RUNTIME-CLOUD-GATEWAY-BYPASS": [
"Route ODS services through LiteLLM so hosted, private-cloud, and auth behavior stay consistent.",
],
"ODS-RUNTIME-LEMONADE-RETIRED": [
"Rerun the installer of this release to move the Lemonade-era settings to the llama.cpp runtime (`ods update` only refreshes images); your model files and active model are kept.",
],
"ODS-RUNTIME-HOST-NATIVE-CLOUD-OVERLAY-CONFLICT": [
"Regenerate compose flags without docker-compose.cloud.yml; docker-compose.host-native-llm.yml disables only the in-stack llama-server and keeps model-router.",
],
"ODS-RUNTIME-HOST-NATIVE-OVERLAY-MISSING": [
"Rerun setup from the ODS Portal so the compose stack includes docker-compose.host-native-llm.yml.",
],
"ODS-RUNTIME-HOST-NATIVE-LOCAL-ROUTE": [
"Route clients through LiteLLM, usually http://litellm:4000, which holds the Windows llama-server's key.",
],
"ODS-RUNTIME-HOST-NATIVE-KEY-MISSING": [
"Rerun setup from the ODS Portal; it passes the llama-server key with --native-llm-api-key-env.",
],
"ODS-RUNTIME-LOCAL-CLOUD-OVERLAY": [
"Regenerate compose flags for local mode so local inference starts normally.",
],
"ODS-RUNTIME-LOCAL-LITELLM-ROUTE": [
"Set LLM_API_URL back to http://llama-server:8080 unless you route local inference through LiteLLM on purpose.",
],
"ODS-RUNTIME-MODE-UNKNOWN": [
"Set ODS_MODE to local, cloud, or hybrid.",
],
"ODS-RUNTIME-VLLM-OVERLAY-MISSING": [
"Run `ods restart`: the CLI resolves the compose stack again from LLM_BACKEND=vllm.",
],
"ODS-RUNTIME-VLLM-LLAMA-ROUTE": [
"Run `ods mode local` to point LLM_API_URL at the LiteLLM gateway, then `ods restart`.",
],
}
for issue in issues:
diagnoses.append(
_diagnosis(
issue["id"],
issue["severity"],
"high" if issue["severity"] == "blocker" else "medium",
issue_titles.get(issue["id"], "Inference runtime contract mismatch"),
[_evidence(issue["source"], issue["detail"])],
"ODS services can report healthy while chat, agents, or model routing target the wrong inference owner.",
next_steps.get(issue["id"], ["Regenerate .env/.compose-flags with the installer and restart ODS."]),
)
)
return {
"ods_mode": ods_mode,
"gpu_backend": gpu_backend,
"llm_backend": llm_backend,
"expected_inference_owner": expected_owner,
"expected_gateway": expected_gateway,
"host_native_llm": host_native,
"llm_api_url": llm_api_url,
"hermes_llm_base_url": hermes_base_url,
"compose_files": compose_files,
"signals": {
"compose_flags_exists": compose_flags_exists,
"cloud_overlay": cloud_overlay,
"host_native_overlay": host_native_overlay,
"local_inference_overlay": local_inference_overlay,
"native_key_configured": native_key_configured,
"vllm_engine": vllm_engine,
"vllm_overlay": vllm_overlay,
},
"issues": issues,
"issue_counts": {
"blockers": sum(1 for item in issues if item["severity"] == "blocker"),
"warnings": sum(1 for item in issues if item["severity"] == "warn"),
},
"diagnoses": diagnoses,
}
def _extract_image(text):
match = re.search(
r"failed to resolve reference\s+\"?([^\"\s]+:[A-Za-z0-9._-]+)\"?",
text,
flags=re.IGNORECASE,
)
if match:
return match.group(1)
match = re.search(
r"([a-z0-9._-]+(?:[.:][0-9]+)?/[a-z0-9._/-]+:[A-Za-z0-9._-]+).*?(?:not found|manifest unknown)",
text,
flags=re.IGNORECASE | re.DOTALL,
)
return match.group(1) if match else ""
def _is_windows_alpine_probe_failure(text):
lowered = text.lower()
has_alpine_probe = (
"alpine:3.20" in lowered
or "unable to find image 'alpine" in lowered
or 'unable to find image "alpine' in lowered
)
if not has_alpine_probe:
return False
return (
"docker could not download" in lowered
or "bind-mount probe" in lowered
or "file-sharing probe" in lowered
or "cannot bind-mount" in lowered
)
def _collect_install_artifacts():
latest_report = _latest_install_report()
compose_launch = root_dir / "logs" / "compose-launch.txt"
compose_up = root_dir / "logs" / "compose-up.log"
install_log = root_dir / "logs" / "install.log"
env_file = root_dir / ".env"
compose_flags = root_dir / ".compose-flags"
return {
"root_dir": root_dir.as_posix(),
"env_file": {
"path": _source(env_file),
"exists": env_file.is_file(),
},
"compose_flags_file": {
"path": _source(compose_flags),
"exists": compose_flags.is_file(),
},
"compose_launch_record": {
"path": _source(compose_launch),
"exists": compose_launch.is_file(),
},
"compose_up_log": {
"path": _source(compose_up),
"exists": compose_up.is_file(),
},
"install_log": {
"path": _source(install_log),
"exists": install_log.is_file(),
},
"latest_install_report": (
{
"path": _source(latest_report),
"exists": True,
}
if latest_report
else {"path": None, "exists": False}
),
"current_ods_containers": {
"managed": _int_arg(ods_managed_container_count),
"running": _int_arg(ods_running_container_count),
},
}
def _collect_install_diagnoses(artifacts):
diagnoses = []
env_path = root_dir / ".env"
flags_path = root_dir / ".compose-flags"
compose_launch_path = root_dir / "logs" / "compose-launch.txt"
compose_up_path = root_dir / "logs" / "compose-up.log"
install_log_path = root_dir / "logs" / "install.log"
latest_report_path = _latest_install_report()
installed_like = (
flags_path.exists()
or compose_launch_path.exists()
or (root_dir / "data").exists()
or bool(latest_report_path)
)
if installed_like and not env_path.exists():
diagnoses.append(
_diagnosis(
"ODS-INSTALL-ENV-MISSING",
"blocker",
"high",
"Install directory is missing .env",
[
_evidence(_source(env_path), "Expected installer-generated .env was not found."),
_evidence(artifacts["root_dir"], "Doctor root appears to be an installed ODS tree."),
],
"Docker Compose and service config can resolve missing or default values, causing startup failures.",
[
"Re-run the installer from the source checkout.",
"Run ODS commands from the installed directory that contains .env.",
],
)
)
if compose_launch_path.exists():
launch_text = _read_text(compose_launch_path)
launch = _parse_kv_lines(launch_text)
launch_cwd = launch.get("cwd")
if launch_cwd:
try:
launch_cwd_path = pathlib.Path(launch_cwd).resolve()
except OSError:
launch_cwd_path = pathlib.Path(launch_cwd)
if launch_cwd_path != root_dir:
diagnoses.append(
_diagnosis(
"ODS-COMPOSE-CWD-MISMATCH",
"blocker",
"high",
"Compose launch record points at a different working directory",
[
_evidence(_source(compose_launch_path), f"cwd={launch_cwd}"),
_evidence("doctor.root_dir", root_dir.as_posix()),
],
"Compose can miss .env or use the wrong relative compose files when launched outside the install directory.",
[
f"Run ODS commands from {root_dir.as_posix()}.",
"Use `ods start` / `ods restart` instead of raw docker compose from a source checkout.",
],
)
)
failure_sources = []
if latest_report_path:
failure_sources.append((latest_report_path, _read_text(latest_report_path)))
if compose_up_path.exists():
failure_sources.append((compose_up_path, _read_text(compose_up_path)))
if install_log_path.exists():
failure_sources.append((install_log_path, _read_text(install_log_path)))
combined_failure_text = "\n".join(text for _, text in failure_sources if text)
if combined_failure_text:
failed_image = _extract_image(combined_failure_text)
if failed_image:
evidence = []
if latest_report_path:
evidence.append(_evidence(_source(latest_report_path), f"Failed image: {failed_image}"))
if compose_up_path.exists():
evidence.append(_evidence(_source(compose_up_path), f"Failed image: {failed_image}"))
diagnoses.append(
_diagnosis(
"ODS-DOCKER-IMAGE-UNRESOLVED",
"blocker",
"high",
"Docker image reference could not be resolved",
evidence,
"The stack cannot start until the image tag is corrected, republished, or replaced by a supported fallback.",
[
f"Check whether `{failed_image}` exists in the registry.",
"Update the image pin or pull a known-good ODS version, then re-run the installer.",
],
)
)
lowered = combined_failure_text.lower()
if _source_mentions_zero_containers(
combined_failure_text
) and not _zero_container_failure_is_stale(failure_sources, compose_launch_path):
diagnoses.append(
_diagnosis(
"ODS-COMPOSE-ZERO-CONTAINERS",
"blocker",
"high",
"Docker Compose completed without creating ODS containers",
[
_evidence(_source(latest_report_path), "Report/log mentions zero managed containers.")
if latest_report_path
else _evidence(_source(compose_up_path), "Compose log mentions zero managed containers.")
],
"Startup may appear to finish while no service is actually managed by the resolved compose stack.",
[
"Open logs/compose-launch.txt and run the saved ps/logs commands.",
"Fix the compose/runtime error and re-run the installer.",
],
)
)
if "modulenotfounderror: no module named 'yaml'" in lowered or "pyyaml is required" in lowered:
diagnoses.append(
_diagnosis(
"ODS-PYTHON-PYYAML-MISSING",
"blocker",
"high",
"Selected Python cannot import PyYAML",
[
_evidence(
_source(latest_report_path) if latest_report_path else _source(compose_up_path),
"Failure text mentions missing yaml/PyYAML.",
)
],
"Compose resolution and generated config validation depend on PyYAML and can fail before services start.",
[
"Re-run the installer so it can install PyYAML or create the private installer venv.",
"If using Conda/venv, deactivate it and retry with the system Python.",
],
)
)
alpine_probe_source = None
for source_path, source_text in failure_sources:
if _is_windows_alpine_probe_failure(source_text):
alpine_probe_source = source_path
break
if alpine_probe_source:
diagnoses.append(
_diagnosis(
"ODS-WINDOWS-FILE-SHARING-PROBE-IMAGE",
"warn",
"medium",
"Windows bind-mount probe was blocked before the file-sharing check completed",
[
_evidence(
_source(alpine_probe_source),
"Failure text includes Alpine probe image pull/download or bind-mount probe failure.",
)
],
"Docker Desktop can report a file-sharing failure when the real prerequisite is the probe image/network path.",
[
"Run `docker pull alpine:3.20` and verify Docker Desktop has internet access.",
"Then verify Docker Desktop > Settings > Resources > File Sharing includes the install directory.",
],
)
)
# Deduplicate by id while preserving order; one diagnosis per root cause keeps
# the human output stable and avoids noisy repeats across install report/log.
seen = set()
unique = []
for item in diagnoses:
if item["id"] in seen:
continue
seen.add(item["id"])
unique.append(item)
return unique
install_artifacts = _collect_install_artifacts()
inference_contract = _collect_inference_contract()
diagnoses = _collect_install_diagnoses(install_artifacts) + inference_contract.get("diagnoses", [])
inference_contract_public = dict(inference_contract)
inference_contract_public.pop("diagnoses", None)
report = {
"version": "1",
"generated_at": datetime.now(timezone.utc).isoformat(),
"autofix_hints": [],
"capability_profile": cap,
"preflight": pre,
"install_artifacts": install_artifacts,
"diagnoses": diagnoses,
"runtime": {
"docker_cli": docker_cli == "true",
"docker_daemon": docker_daemon == "true",
"docker_access_denied": docker_access_denied == "true",
"compose_cli": compose_cli == "true",
"dashboard_http": dashboard_http == "true",
"webui_enabled": webui_enabled == "true",
"webui_http": webui_http == "true",
"llm_backend": {
"status": os.environ.get("LLM_STATUS", "unknown"),
"provider": os.environ.get("LLM_PROVIDER", ""),
"url": os.environ.get("LLM_URL", ""),
"model": os.environ.get("LLM_MODEL", ""),
"local_running_warning": os.environ.get("LLM_LOCAL_WARNING", "false") == "true",
"recovery_hint": os.environ.get("LLM_RECOVERY", ""),
},
"stt_model_cached": stt_cached,
"stt_model_name": stt_model_name,
"tts_http": tts_http,
"tts_port": tts_port,
"dgx_spark_gpu": dgx_spark_gpu == "true",
"dgx_spark_gpu_name": dgx_spark_gpu_name,
"dgx_spark_compute_cap": dgx_spark_compute_cap,
"llama_cuda_archs": llama_cuda_archs,
"dgx_spark_cuda_arch_check": {
"status": dgx_spark_arch_status,
"message": dgx_spark_arch_message,
},
"hermes_slash_workers": {
"count": hermes_slash_worker_count_num,
"max_count": hermes_slash_worker_max_count_num,
"status": "warn"
if hermes_slash_worker_count_num > hermes_slash_worker_max_count_num
else "pass",
},
"rootless_docker": os.environ.get("ROOTLESS_DOCKER") == "true",
"rootless_subid_check": {
"status": os.environ.get("ROOTLESS_SUBID_STATUS", "skipped"),
"user": os.environ.get("ROOTLESS_SUBID_USER", ""),
"subuid_total": _int_value(os.environ.get("ROOTLESS_SUBUID_TOTAL")),
"subgid_total": _int_value(os.environ.get("ROOTLESS_SUBGID_TOTAL")),
},
"amd_runtime": amd_runtime,
"inference_contract": inference_contract_public,
},
"extensions": ext_diagnostics,
"summary": {
"preflight_blockers": pre.get("summary", {}).get("blockers", 0),
"preflight_warnings": pre.get("summary", {}).get("warnings", 0),
"runtime_warnings": (
(1 if dgx_spark_arch_status == "warn" else 0)
+ (1 if stt_cached in {"false", "service_down"} else 0)
+ (1 if tts_http == "false" else 0)
+ (1 if hermes_slash_worker_count_num > hermes_slash_worker_max_count_num else 0)
+ (1 if os.environ.get("ROOTLESS_SUBID_STATUS") in {"warn", "fail"} else 0)
+ len(amd_runtime.get("warnings", []))
+ inference_contract.get("issue_counts", {}).get("warnings", 0)
),
"runtime_contract_blockers": inference_contract.get("issue_counts", {}).get("blockers", 0),
"runtime_contract_warnings": inference_contract.get("issue_counts", {}).get("warnings", 0),
"diagnoses_total": len(diagnoses),
"diagnoses_blockers": sum(1 for d in diagnoses if d.get("severity") == "blocker"),
"diagnoses_warnings": sum(1 for d in diagnoses if d.get("severity") == "warn"),
"runtime_ready": (docker_daemon == "true" and compose_cli == "true"),
"extensions_total": len(ext_diagnostics),
"extensions_healthy": sum(1 for e in ext_diagnostics if e.get("health_status") == "healthy"),
"extensions_issues": sum(1 for e in ext_diagnostics if len(e.get("issues", [])) > 0),
},
}
fix_hints = []
for diagnosis in diagnoses:
fix_hints.extend(diagnosis.get("next_steps", []))
for check in pre.get("checks", []):
status = check.get("status")
action = (check.get("action") or "").strip()
if status in {"blocker", "warn"} and action:
fix_hints.append(action)
runtime = report["runtime"]
if not runtime["docker_cli"]:
fix_hints.append("Install Docker CLI/Docker Desktop and reopen your terminal.")
if runtime["docker_cli"] and runtime["docker_access_denied"]:
fix_hints.append("Docker is running, but this account cannot use it yet. Log out and back in "
"(or run 'newgrp docker') so the docker group membership applies, then run "
"ods doctor again.")
elif runtime["docker_cli"] and not runtime["docker_daemon"]:
fix_hints.append("Start Docker daemon/Desktop before launching ODS.")
if not runtime["compose_cli"]:
fix_hints.append("Install Docker Compose v2 plugin (or docker-compose).")
if runtime["docker_daemon"] and not runtime["dashboard_http"]:
fix_hints.append(f"Run installer/start command, then verify dashboard on http://127.0.0.1:{dashboard_port}.")
if runtime["docker_daemon"] and runtime["webui_enabled"] and not runtime["webui_http"]:
fix_hints.append(f"Verify Open WebUI container and port {webui_port} mapping.")
llm_status = os.environ.get("LLM_STATUS", "unknown")
llm_recovery = os.environ.get("LLM_RECOVERY", "")
llm_provider = os.environ.get("LLM_PROVIDER", "")
llm_local_warn = os.environ.get("LLM_LOCAL_WARNING", "false") == "true"
llm_failure = os.environ.get("LLM_FAILURE", "")
if llm_status == "fail" and llm_recovery:
if llm_provider == "vLLM":
fix_hints.append(f"The vLLM engine (vllm-server) is not serving the model. Hint: {llm_recovery}")
elif llm_provider != "llama-server" and llm_failure in ("key-refused", "key-required"):
fix_hints.append(f"External LLM backend ({llm_provider}) answered but did not accept a stored API key. Hint: {llm_recovery}")
elif llm_provider != "llama-server":
fix_hints.append(f"External LLM backend ({llm_provider}) is unreachable. Hint: {llm_recovery}")
else:
fix_hints.append(f"Local llama-server is unreachable. Hint: {llm_recovery}")
if llm_local_warn and llm_provider == "vLLM":
# `ods stop llama-server` names vllm-server on this route; stop the container itself.
fix_hints.append("A llama-server container from an earlier install is running beside the vLLM engine "
"and competes for its GPU. Stop it with: docker stop ods-llama-server")
elif llm_local_warn:
fix_hints.append("External LLM is configured but the local llama-server container is still running. Stop it with: ods stop llama-server")
# Rootless daemon without (enough) subordinate IDs: images fail to extract
# with "uid/gid map out of range" long after doctor reports healthy Docker.
# Two hints: the fixed 100000-165535 range is only safe to suggest when the
# user has NO allocation; with an existing-but-small range that command
# could overlap what is already allocated, so ask for a non-overlapping
# extension instead.
_rootless_subid = report["runtime"]["rootless_subid_check"]
if _rootless_subid["status"] == "fail":
fix_hints.append(
"Rootless Docker: no subordinate ID allocation for "
f"{_rootless_subid['user']} "
f"(subuids={_rootless_subid['subuid_total']}, subgids={_rootless_subid['subgid_total']}). "
f"Run: sudo usermod --add-subuids 100000-165535 --add-subgids 100000-165535 {_rootless_subid['user']} "
"then restart the rootless Docker daemon (systemctl --user restart docker)."
)
elif _rootless_subid["status"] == "warn":
fix_hints.append(
"Rootless Docker: subordinate ID allocation for "
f"{_rootless_subid['user']} is below 65536 "
f"(subuids={_rootless_subid['subuid_total']}, subgids={_rootless_subid['subgid_total']}). "
"Extend it to at least 65536 IDs with a range that does not overlap "
"existing entries in /etc/subuid and /etc/subgid "
f"(sudo usermod --add-subuids <start>-<start+65535> --add-subgids <start>-<start+65535> {_rootless_subid['user']}), "
"then restart the rootless Docker daemon (systemctl --user restart docker)."
)
# STT model cache: service up but model missing is a common silent failure
if stt_cached == "false" and stt_recovery:
fix_hints.append(
f"Whisper STT model '{stt_model_name}' not cached — transcription will 404. "
f"Run: {stt_recovery}"
)
elif stt_cached == "service_down":
fix_hints.append("Whisper STT is not responding. Run: ods repair voice")
if tts_http == "false":
fix_hints.append("Kokoro TTS is not responding. Run: ods repair voice")
if hermes_slash_worker_count_num > hermes_slash_worker_max_count_num:
fix_hints.append(
f"Hermes has {hermes_slash_worker_count_num} slash_worker children "
f"(policy max {hermes_slash_worker_max_count_num}). Run: ods repair hermes-workers"
)
if dgx_spark_arch_status == "warn":
fix_hints.append(
"This GPU (DGX Spark GB10 or RTX Spark N1X, compute capability 12.1) runs a llama-server without "
"sm_121 kernels, so its answers can come back as '????'. Rerun the ODS installer: it selects the "
"CUDA 13 llama.cpp image (server-cuda13) pinned for this GPU. If you set LLAMA_SERVER_IMAGE yourself, "
"use that image or a llama.cpp build made with -DGGML_CUDA=ON -DCMAKE_CUDA_ARCHITECTURES=121."
)
for warning in amd_runtime.get("warnings", []):
if warning == "health_unreachable":
fix_hints.append("AMD inference runtime is configured but its health endpoint is unreachable. Start the runtime or run 'ods restart'.")
elif warning == "amd_supported_backends_env_missing":
fix_hints.append("AMD runtime capabilities are missing from .env. Re-run the installer or add AMD_INFERENCE_SUPPORTED_BACKENDS.")
elif warning == "amd_selected_backend_not_supported":
fix_hints.append("AMD_INFERENCE_BACKEND is not listed in AMD_INFERENCE_SUPPORTED_BACKENDS. Check the installer-generated .env.")
# Extension-specific hints
for ext in ext_diagnostics:
ext_id = ext.get("id", "unknown")
container_state = ext.get("container_state", "unknown")
issues = ext.get("issues", [])
for issue in issues:
if issue == "container_not_running":
if container_state == "not_found":
fix_hints.append(f"Extension {ext_id}: not installed (image not built). Skipped by installer or disabled by tier system.")
else:
fix_hints.append(f"Extension {ext_id}: container not running. Run 'ods start {ext_id}'.")
elif issue == "health_check_failed":
fix_hints.append(f"Extension {ext_id}: health check failed. Check logs with 'docker logs ods-{ext_id}'.")
elif issue == "gpu_backend_incompatible":
fix_hints.append(f"Extension {ext_id}: incompatible with current GPU backend. Consider disabling.")
elif issue.startswith("missing_dependency:"):
dep = issue.split(":", 1)[1]
fix_hints.append(f"Extension {ext_id}: missing dependency '{dep}'. Run 'ods enable {dep}'.")
# Deduplicate while preserving order
seen = set()
uniq_hints = []
for hint in fix_hints:
if hint in seen:
continue
seen.add(hint)
uniq_hints.append(hint)
report["autofix_hints"] = uniq_hints # overwrite initial empty list
path = pathlib.Path(report_file)
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8")
PY
echo "ODS Doctor report: $REPORT_FILE"
echo " Preflight blockers: ${PREFLIGHT_BLOCKERS:-0}"
echo " Preflight warnings: ${PREFLIGHT_WARNINGS:-0}"
echo " Docker daemon: $DOCKER_DAEMON"
if [[ "$DOCKER_ACCESS_DENIED" == "true" ]]; then
echo " Docker access: permission denied for $(id -un); log out and back in (or run 'newgrp docker')"
fi
echo " Compose CLI: $COMPOSE_CLI"
"$PYTHON_CMD" - "$REPORT_FILE" <<'PY'
import json
import sys
path = sys.argv[1]
try:
data = json.load(open(path, "r", encoding="utf-8"))
except Exception:
raise SystemExit(0)
# Show extension summary
summary = data.get("summary", {})
ext_total = summary.get("extensions_total", 0)
ext_healthy = summary.get("extensions_healthy", 0)
ext_issues = summary.get("extensions_issues", 0)
if ext_total > 0:
print(f" Extensions: {ext_healthy}/{ext_total} healthy, {ext_issues} with issues")
dgx_check = data.get("runtime", {}).get("dgx_spark_cuda_arch_check", {})
if dgx_check.get("status") == "warn":
print(f" Spark GPU: warning - {dgx_check.get('message')}")
elif dgx_check.get("status") == "pass":
print(" Spark GPU: llama-server includes sm_121 kernels")
amd_runtime = data.get("runtime", {}).get("amd_runtime", {})
if amd_runtime.get("available"):
print(
" AMD Runtime: "
f"{amd_runtime.get('runtime')} / {amd_runtime.get('selectedBackend')} / "
f"{amd_runtime.get('location')} / {amd_runtime.get('health')}"
)
elif amd_runtime.get("reason") not in (None, "", "not_amd", "api_mode"):
print(f" AMD Runtime: {amd_runtime.get('reason')}")
hermes_workers = data.get("runtime", {}).get("hermes_slash_workers", {})
if hermes_workers.get("status") == "warn":
print(
" Hermes: "
f"{hermes_workers.get('count')} slash_worker processes "
f"(policy max {hermes_workers.get('max_count')})"
)
diagnoses = data.get("diagnoses") or []
if diagnoses:
blockers = sum(1 for item in diagnoses if item.get("severity") == "blocker")
warnings = sum(1 for item in diagnoses if item.get("severity") == "warn")
print(f" Diagnoses: {len(diagnoses)} total, {blockers} blocker(s), {warnings} warning(s)")
for item in diagnoses[:5]:
print(f" - {item.get('id')}: {item.get('title')}")
hints = data.get("autofix_hints") or []
if hints:
print(" Suggested fixes:")
for hint in hints[:10]:
print(f" - {hint}")
# The ODS community Discord (ODS_HELP_DISCORD_URL in installers/lib/constants.sh).
if diagnoses or hints:
print(" Need help? Ask on the ODS Discord: https://discord.gg/4ntNp9MAwC")
print(" Attach a redacted support bundle: scripts/ods-support-bundle.sh")
PY