Skip to content

armgit / ODS

Public Imported from GitHub
0

ODS V3 Pre-Release: Public testing and refinement ahead of the official V3 launch. Turn your PC, Mac, or Linux box into a private AI server.

18.7 KB
Raw Blame
"""GPU router — per-GPU metrics, topology, and rolling history."""

import asyncio
import json
import logging
import os
import time
import urllib.error
import urllib.request
from collections import deque
from datetime import datetime, timezone
from typing import Optional

from fastapi import APIRouter, Depends, HTTPException

from security import verify_api_key

from helpers import get_cached_services
from gpu import (
    aggregate_gpu_details,
    decode_gpu_assignment,
    get_gpu_info_amd_detailed,
    get_gpu_info_apple,
    get_gpu_info_nvidia_detailed,
    get_gpu_info_windows_host,
    get_gpu_info_windows_host_detailed,
    get_gpu_info_wsl_host_detailed,
    get_wsl_gpu_backend,
    _live_env_value,
    read_gpu_topology,
)
from models import GPUInfo, IndividualGPU, MultiGPUStatus
from models import AmdRuntimeStatus

logger = logging.getLogger(__name__)

router = APIRouter(tags=["gpu"])

# Rolling history buffer — 60 samples max (5 min at 5 s intervals)
_GPU_HISTORY: deque = deque(maxlen=60)
_HISTORY_POLL_INTERVAL = 5.0

# Simple per-endpoint TTL caches
_detailed_cache: dict = {"expires": 0.0, "value": None}
_detailed_lock = asyncio.Lock()
_topology_cache: dict = {"expires": 0.0, "value": None}
_GPU_DETAILED_TTL = 3.0
_GPU_TOPOLOGY_TTL = 300.0


# ============================================================================
# Internal helpers
# ============================================================================

def _apple_info_to_individual(info: GPUInfo) -> IndividualGPU:
    """Wrap an Apple Silicon aggregate GPUInfo as a single IndividualGPU entry."""
    return IndividualGPU(
        index=0,
        uuid="apple-unified-0",  # 15 chars; GPUCard.jsx calls uuid.slice(-8)
        name=info.name,
        memory_used_mb=info.memory_used_mb,
        memory_total_mb=info.memory_total_mb,
        memory_percent=info.memory_percent,
        utilization_percent=info.utilization_percent,
        temperature_c=info.temperature_c,
        power_w=info.power_w,
        memory_type="unified",
        assigned_services=[],
        memory_usage_available=info.memory_usage_available,
        utilization_available=info.utilization_available,
        temperature_available=info.temperature_available,
    )


def _get_raw_gpus(gpu_backend: str) -> Optional[list[IndividualGPU]]:
    """Return per-GPU list from the appropriate backend, with fallback."""
    if gpu_backend == "apple":
        info = get_gpu_info_apple()
        if info is None:
            return None
        return [_apple_info_to_individual(info)]
    if gpu_backend == "amd":
        result = get_gpu_info_amd_detailed()
        if result:
            return result
        return _amd_host_runtime_fallback_gpus()
    result = get_gpu_info_nvidia_detailed()
    if result:
        return result
    return get_gpu_info_amd_detailed() or get_gpu_info_wsl_host_detailed()


def _env_int(name: str, default: int = 0) -> int:
    raw = os.environ.get(name, "").strip()
    if not raw:
        return default
    try:
        return int(raw)
    except ValueError:
        return default


def _amd_host_runtime_fallback_gpus() -> Optional[list[IndividualGPU]]:
    """Represent a healthy host-backed AMD runtime when container GPU sysfs is absent.

    Windows Docker Desktop installs route inference through a host
    llama-server process. In that mode dashboard-api cannot read AMD DRM sysfs
    from inside the Linux container, but the runtime is still configured and
    usable. Return a conservative capability/status object instead of 503.
    """
    runtime = _amd_runtime_name(_clean_env("AMD_INFERENCE_RUNTIME"))
    location = _clean_env("AMD_INFERENCE_LOCATION").lower()
    runtime_mode = _clean_env("AMD_INFERENCE_RUNTIME_MODE").lower()
    if runtime != "llama-server" or location != "host":
        return None
    if not runtime_mode.startswith("windows"):
        return None

    detailed = get_gpu_info_windows_host_detailed()
    if detailed:
        return detailed

    host_info = get_gpu_info_windows_host()
    if host_info is not None:
        return [IndividualGPU(
            index=0,
            uuid="amd-windows-host-0",
            name=host_info.name,
            memory_used_mb=host_info.memory_used_mb,
            memory_total_mb=host_info.memory_total_mb,
            memory_percent=host_info.memory_percent,
            utilization_percent=host_info.utilization_percent,
            temperature_c=host_info.temperature_c,
            power_w=host_info.power_w,
            memory_type=host_info.memory_type,
            assigned_services=["llama-server"],
            memory_usage_available=host_info.memory_usage_available,
            utilization_available=host_info.utilization_available,
            temperature_available=host_info.temperature_available,
        )]

    count = max(1, _env_int("GPU_COUNT", 1))
    backend = _clean_env("AMD_INFERENCE_BACKEND").lower() or "unknown"
    name = "AMD llama-server host runtime"
    if backend not in {"", "unknown"}:
        name = f"{name} ({backend})"

    return [
        IndividualGPU(
            index=idx,
            uuid=f"amd-host-runtime-{idx}",
            name=name,
            memory_used_mb=0,
            memory_total_mb=0,
            memory_percent=0.0,
            utilization_percent=0,
            temperature_c=0,
            power_w=None,
            memory_type="discrete",
            assigned_services=["llama-server"],
            memory_usage_available=False,
            utilization_available=False,
            temperature_available=False,
        )
        for idx in range(count)
    ]


def _clean_env(name: str) -> str:
    return os.environ.get(name, "").strip()


def _amd_runtime_name(value: str) -> str:
    """Every managed AMD runtime is llama-server; ``lemonade`` reads as it for one release."""
    runtime = value.strip().lower()
    return "llama-server" if runtime == "lemonade" else runtime


def _join_url(base_url: str, path: str) -> str:
    base = base_url.rstrip("/")
    suffix = path if path.startswith("/") else f"/{path}"
    return f"{base}{suffix}"


def _runtime_port() -> tuple[int, Optional[str]]:
    raw = _clean_env("AMD_INFERENCE_PORT")
    if not raw:
        return 8080, None
    try:
        port = int(raw)
    except ValueError:
        return 8080, "amd_port_invalid"
    if 1 <= port <= 65535:
        return port, None
    return 8080, "amd_port_invalid"


def _split_backend_list(raw: str) -> tuple[list[str], Optional[str]]:
    if not raw:
        return [], None

    backends: list[str] = []
    invalid: list[str] = []
    for item in raw.split(","):
        backend = item.strip().lower()
        if not backend:
            continue
        if backend in {"auto", "cpu", "npu", "rocm", "vulkan"}:
            if backend not in backends:
                backends.append(backend)
        else:
            invalid.append(backend)
    if invalid:
        return backends, "amd_supported_backends_invalid"
    return backends, None


def _env_bool(name: str) -> bool:
    return _clean_env(name).lower() in {"1", "true", "yes", "on"}


def _runtime_base_url(location: str, port: int) -> str:
    if location == "host":
        # The Windows-hosted server as containers reach it; the legacy key
        # name is read for one release.
        configured = _clean_env("NATIVE_LLM_CONTAINER_BASE_URL") or _clean_env("LEMONADE_CONTAINER_BASE_URL")
        if configured:
            configured = configured.rstrip("/")
            for suffix in ("/api/v1", "/v1", "/api"):
                if configured.endswith(suffix):
                    return configured[: -len(suffix)]
            return configured
        return f"http://host.docker.internal:{port}"
    if location == "container":
        return f"http://llama-server:{port}"
    for key in ("OLLAMA_URL", "LLM_URL", "LLM_API_URL"):
        value = _clean_env(key)
        if value and "litellm" not in value.lower():
            return value
    return "http://llama-server:8080"


def _probe_amd_health(health_url: str) -> tuple[str, str, Optional[str]]:
    request = urllib.request.Request(health_url, method="GET")
    try:
        with urllib.request.urlopen(request, timeout=2.0) as response:
            status = getattr(response, "status", response.getcode())
            body = response.read(4096).decode("utf-8", errors="replace")
    except urllib.error.HTTPError as exc:
        return "unhealthy", "unknown", f"health_http_{exc.code}"
    except (urllib.error.URLError, TimeoutError, OSError) as exc:
        logger.debug("AMD runtime health probe failed for %s: %s", health_url, exc)
        return "unreachable", "unknown", "health_unreachable"

    version = "unknown"
    try:
        payload = json.loads(body) if body else {}
        if isinstance(payload, dict) and payload.get("version"):
            version = str(payload["version"])
    except json.JSONDecodeError:
        pass

    if 200 <= int(status) < 300:
        return "reachable", version, None
    return "unhealthy", version, f"health_http_{status}"


# ============================================================================
# Endpoints
# ============================================================================

@router.get("/api/gpu/detailed", response_model=MultiGPUStatus, dependencies=[Depends(verify_api_key)])
async def gpu_detailed():
    """Per-GPU metrics with service assignment info (cached 3 s)."""
    # Multiple dashboard clients can miss the same cache entry. Recheck under
    # the lock so only one successful probe populates each new snapshot.
    async with _detailed_lock:
        if time.monotonic() < _detailed_cache["expires"] and _detailed_cache["value"] is not None:
            return _detailed_cache["value"]
        result = await _read_detailed_gpu_status()
        _detailed_cache["value"] = result
        _detailed_cache["expires"] = time.monotonic() + _GPU_DETAILED_TTL
        return result


def _drop_services_off_the_gpu(gpus: list[IndividualGPU]) -> None:
    """Keep a GPU's assigned services only when they run on a GPU here.

    The installer assigns a GPU to every GPU-capable service, enabled or not,
    and Whisper on the CPU image keeps its assignment. Before the first status
    poll the assignment is all there is.
    """
    statuses = get_cached_services()
    if statuses is None:
        return
    deployed = {status.id for status in statuses if status.status != "not_deployed"}
    whisper_on_cpu = _live_env_value("WHISPER_ACCELERATION").strip().lower() == "cpu"
    for gpu in gpus:
        # Assignment names use underscores (llama_server); service IDs use hyphens.
        gpu.assigned_services = [
            name for name in gpu.assigned_services
            if name.replace("_", "-") in deployed and not (name == "whisper" and whisper_on_cpu)
        ]


async def _read_detailed_gpu_status() -> MultiGPUStatus:
    """Discover one complete GPU snapshot without publishing partial results."""
    gpu_backend = os.environ.get("GPU_BACKEND", "").lower() or "nvidia"
    gpus = await asyncio.to_thread(_get_raw_gpus, gpu_backend)
    if not gpus:
        raise HTTPException(status_code=503, detail="No GPU data available")

    if all(gpu.uuid.startswith("luid_0x") for gpu in gpus):
        gpu_backend = await asyncio.to_thread(get_wsl_gpu_backend)
    aggregate = aggregate_gpu_details(gpus, gpu_backend)

    assignment_full = decode_gpu_assignment()
    assignment_data = assignment_full.get("gpu_assignment") if assignment_full else None
    if assignment_full:
        _drop_services_off_the_gpu(gpus)

    return MultiGPUStatus(
        gpu_count=len(gpus),
        backend=gpu_backend,
        gpus=gpus,
        topology=None,  # topology is served from its own endpoint
        assignment=assignment_data,
        split_mode=_live_env_value("LLAMA_ARG_SPLIT_MODE") or None,
        tensor_split=_live_env_value("LLAMA_ARG_TENSOR_SPLIT") or None,
        aggregate=aggregate,
    )


@router.get("/api/gpu/topology", dependencies=[Depends(verify_api_key)])
async def gpu_topology():
    """GPU topology from config/gpu-topology.json (written by installer / ods-cli). Cached 300 s."""
    now = time.monotonic()
    if now < _topology_cache["expires"] and _topology_cache["value"] is not None:
        return _topology_cache["value"]

    topo = await asyncio.to_thread(read_gpu_topology)
    if not topo:
        raise HTTPException(
            status_code=404,
            detail="GPU topology not available. Run 'ods gpu reassign' to generate it.",
        )

    _topology_cache["expires"] = now + _GPU_TOPOLOGY_TTL
    _topology_cache["value"] = topo
    return topo


@router.get(
    "/api/gpu/amd-runtime",
    response_model=AmdRuntimeStatus,
    response_model_exclude_none=True,
    dependencies=[Depends(verify_api_key)],
)
async def amd_runtime():
    """AMD runtime contract and health from explicit installer-provided env."""
    gpu_backend = _clean_env("GPU_BACKEND").lower() or "nvidia"
    if gpu_backend != "amd":
        return AmdRuntimeStatus(
            available=False,
            reason="not_amd",
            runtime="none",
            location="none",
            runtimeMode="none",
            managedByODS=False,
            selectedBackend="none",
            supportedBackends=[],
            defaultBackend="none",
            capabilities=[],
            warnings=[],
        )

    warnings: list[str] = []
    runtime = _clean_env("AMD_INFERENCE_RUNTIME").lower()
    selected_backend = _clean_env("AMD_INFERENCE_BACKEND").lower()
    location = _clean_env("AMD_INFERENCE_LOCATION").lower()
    runtime_mode = _clean_env("AMD_INFERENCE_RUNTIME_MODE").lower()
    managed_raw = _clean_env("AMD_INFERENCE_MANAGED").lower()
    managed_by_ods = _env_bool("AMD_INFERENCE_MANAGED")
    supported_backends, supported_warning = _split_backend_list(
        _clean_env("AMD_INFERENCE_SUPPORTED_BACKENDS")
    )
    if supported_warning:
        warnings.append(supported_warning)

    runtime = _amd_runtime_name(runtime)
    if not runtime:
        legacy_backend = _amd_runtime_name(_clean_env("LLM_BACKEND"))
        if legacy_backend == "llama-server":
            runtime = legacy_backend
            warnings.append("amd_runtime_env_missing")
    if not selected_backend:
        selected_backend = "unknown"
        warnings.append("amd_backend_env_missing")
    if not location:
        location = "unknown"
        warnings.append("amd_location_env_missing")
    if not runtime_mode:
        runtime_mode = "unknown"
        warnings.append("amd_runtime_mode_env_missing")
    if not managed_raw:
        warnings.append("amd_managed_env_missing")
    if not supported_backends:
        warnings.append("amd_supported_backends_env_missing")
    elif selected_backend not in {"", "unknown", "none"} and selected_backend not in supported_backends:
        warnings.append("amd_selected_backend_not_supported")

    if runtime != "llama-server":
        return AmdRuntimeStatus(
            available=False,
            reason="runtime_not_configured",
            runtime=runtime or "none",
            location=location,
            runtimeMode=runtime_mode,
            managedByODS=managed_by_ods,
            selectedBackend=selected_backend,
            supportedBackends=supported_backends,
            defaultBackend=selected_backend or "none",
            capabilities=supported_backends,
            warnings=warnings,
        )

    port, port_warning = _runtime_port()
    if port_warning:
        warnings.append(port_warning)

    # Upstream llama-server: OpenAI routes under /v1, a public /health.
    base_url = _runtime_base_url(location, port)
    api_base = _join_url(base_url, "/v1")
    health_url = _join_url(base_url, "/health")
    loaded_model: Optional[str] = None
    model_count: Optional[int] = None
    health, version, health_warning = await asyncio.to_thread(_probe_amd_health, health_url)
    if health_warning:
        warnings.append(health_warning)

    return AmdRuntimeStatus(
        available=True,
        reason=None,
        runtime=runtime,
        location=location,
        runtimeMode=runtime_mode,
        managedByODS=managed_by_ods,
        selectedBackend=selected_backend,
        supportedBackends=supported_backends,
        defaultBackend=selected_backend or "none",
        apiBase=api_base,
        healthUrl=health_url,
        health=health,
        version=version,
        loadedModel=loaded_model,
        modelCount=model_count,
        capabilities=supported_backends,
        warnings=warnings,
    )


@router.get("/api/gpu/history", dependencies=[Depends(verify_api_key)])
async def gpu_history():
    """Rolling 5-minute per-GPU metrics history sampled every 5 s."""
    if not _GPU_HISTORY:
        return {"timestamps": [], "gpus": {}}

    timestamps = [s["timestamp"] for s in _GPU_HISTORY]

    gpu_keys: set[str] = set()
    for sample in _GPU_HISTORY:
        gpu_keys.update(sample["gpus"].keys())

    gpus_data: dict[str, dict] = {}
    for gpu_key in sorted(gpu_keys):
        gpus_data[gpu_key] = {
            "utilization": [],
            "memory_percent": [],
            "temperature": [],
            "power_w": [],
        }
        for sample in _GPU_HISTORY:
            g = sample["gpus"].get(gpu_key, {})
            gpus_data[gpu_key]["utilization"].append(g.get("utilization"))
            gpus_data[gpu_key]["memory_percent"].append(g.get("memory_percent"))
            gpus_data[gpu_key]["temperature"].append(g.get("temperature"))
            gpus_data[gpu_key]["power_w"].append(g.get("power_w"))

    return {"timestamps": timestamps, "gpus": gpus_data}


# ============================================================================
# Background task
# ============================================================================

async def poll_gpu_history() -> None:
    """Background task: append a per-GPU sample to _GPU_HISTORY every 5 s."""
    while True:
        readings = {}
        try:
            gpu_backend = os.environ.get("GPU_BACKEND", "").lower() or "nvidia"
            gpus = await asyncio.to_thread(_get_raw_gpus, gpu_backend)
            if gpus:
                readings = {
                    str(g.index): {
                        "utilization": g.utilization_percent if g.utilization_available else None,
                        "memory_percent": g.memory_percent if g.memory_usage_available else None,
                        "temperature": g.temperature_c if g.temperature_available else None,
                        "power_w": g.power_w,
                    }
                    for g in gpus
                }
        except Exception:  # Broad catch: background task must survive transient failures
            logger.exception("GPU history poll failed")
        # Advance the bounded window even when discovery is unavailable. The
        # history endpoint represents absent readings as null, never as zero.
        _GPU_HISTORY.append({
            "timestamp": datetime.now(timezone.utc).isoformat(),
            "gpus": readings,
        })
        await asyncio.sleep(_HISTORY_POLL_INTERVAL)