Skip to content

armgit / ODS

Public Imported from GitHub
0

ODS V3 Pre-Release: Public testing and refinement ahead of the official V3 launch. Turn your PC, Mac, or Linux box into a private AI server.

138 KB
Raw Blame
"""Model Library router — browse, benchmark, and manage GGUF models."""

import asyncio
import hashlib
import json
import logging
import math
import os
import re
import tempfile
import threading
import time
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
from typing import Optional
from urllib.parse import quote, urljoin, urlsplit

import httpx
from model_activation_status import model_activation_status
from fastapi import APIRouter, Body, Depends, HTTPException, Query
from fastapi.responses import RedirectResponse, JSONResponse

from env_values import parse_env_value
import hf_gguf_header
import model_preflight
from bootstrap_download_control import valid_generation
from config import (
    DATA_DIR,
    GPU_BACKEND,
    INSTALL_DIR,
    LLM_BACKEND,
    LOCAL_MODEL_MODES,
    ODS_MODE_EFFECTIVE,
    SERVICES,
    normalize_ods_mode,
    read_live_env_values,
)
from gpu import get_gpu_info
from helpers import (
    get_bootstrap_status,
    get_llama_context_size,
    get_llama_metrics,
    get_loaded_model,
    is_plausible_single_request_tps,
    owner_model_config,
    record_model_performance,
)
from host_agent_client import (
    AgentClientError,
    AgentHTTPError,
    AgentProtocolError,
    AgentUnavailable,
    request_json as request_agent_json,
)
from models import ModelLibraryGpu, ModelLibraryResponse
from model_vllm import is_vllm_entry
from pixel_runtime_state import pixel_stream_active
from runtime_projection import active_runtime_projection
from context_policy import HERMES_MIN_CONTEXT
from performance_oracle import (
    activation_context_plan,
    build_models_payload,
    build_sample_signature,
    current_model_matches,
    find_catalog_model,
    load_model_catalog,
    model_files_dir as model_files_dir,
    normalize_catalog_entry,
    planned_model_context,
    read_env_file_value,
    read_env_value,
)
from security import verify_api_key

logger = logging.getLogger(__name__)

router = APIRouter(tags=["models"])

_LIBRARY_PATH = Path(INSTALL_DIR) / "config" / "model-library.json"
_MODELS_DIR = Path(DATA_DIR) / "models"


def _installed_model_paths() -> dict[str, Path]:
    from model_stores import scan_model_files
    return scan_model_files(Path(DATA_DIR), container=Path("/.dockerenv").exists(), default_dir=_MODELS_DIR)


def _installed_model_path(filename: str) -> Path | None:
    return next((path for name, path in _installed_model_paths().items() if name.casefold() == filename.casefold()), None)
_ENV_PATH = Path(INSTALL_DIR) / ".env"


def _windows_hosted_runtime() -> bool:
    """The model runs on the Windows host, controlled only through the host agent.

    The WSL Portal drives an ODS-owned llama-server.exe through the WSL
    bridge; the agent's ownership proof decides whether this install may
    change or stop it. ``LEMONADE_HOST_TRANSPORT`` is the transport key's
    one-release legacy name. An unmigrated pre-round-F .env for the owner's
    own Lemonade (``LLM_BACKEND=lemonade`` with ``LEMONADE_EXTERNAL=true``)
    is never controllable from here.
    """
    env = read_live_env_values((
        "ODS_HOST_LLM_TRANSPORT", "LEMONADE_HOST_TRANSPORT", "AMD_INFERENCE_RUNTIME_MODE",
        "LLM_BACKEND", "LEMONADE_EXTERNAL",
    ))
    transport = str(env.get("ODS_HOST_LLM_TRANSPORT") or env.get("LEMONADE_HOST_TRANSPORT") or "")
    unmigrated_external = (
        str(env.get("LLM_BACKEND") or "").strip().casefold() == "lemonade"
        and str(env.get("LEMONADE_EXTERNAL") or "").strip().casefold() in {"1", "true", "yes", "on"}
    )
    return (
        transport.strip().casefold() == "model-router"
        or str(env.get("AMD_INFERENCE_RUNTIME_MODE") or "").strip().casefold() == "windows-portal-llama-server"
        or unmigrated_external
    )


_HF_API_BASE = "https://huggingface.co"
_HF_REPO_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,95}/[A-Za-z0-9][A-Za-z0-9._-]{0,95}$")
_HF_AUTHOR_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,95}$")
_HF_AVATAR_HOSTS = {"huggingface.co", "cdn-avatars.huggingface.co"}
_HF_SPLIT_GGUF_RE = re.compile(r"^(?P<prefix>.+)-(?P<part>\d{5})-of-(?P<total>\d{5})\.gguf$", re.IGNORECASE)
# MXFP4 is gpt-oss's native format ("gpt-oss-20b-MXFP4.gguf", unsloth's
# "*-MXFP4_MOE.gguf"); without it the import was named "· unknown".
_HF_QUANT_RE = re.compile(
    r"(?:^|[-_.])(?P<quant>(?:IQ\d(?:_[A-Z0-9]+)+|Q\d(?:_[A-Z0-9]+)+|MXFP4(?:_MOE)?|BF16|F16|F32))(?:[-_.]|$)",
    re.IGNORECASE,
)
_HF_SEARCH_CACHE_TTL_SECONDS = 300.0
_HF_SEARCH_CACHE_MAX_ENTRIES = 128
_HF_SEARCH_CACHE: dict[tuple[str, str, int, str], tuple[float, dict[str, Any]]] = {}
_HF_SEARCH_CACHE_LOCK = threading.Lock()
_HF_AVATAR_CACHE_TTL_SECONDS = 3600.0
_HF_AVATAR_CACHE_MAX_ENTRIES = 512
_HF_AVATAR_CACHE: dict[tuple[str, str], tuple[float, str | None]] = {}
_HF_AVATAR_CACHE_LOCK = threading.Lock()
_IMPORTED_MODELS_LOCK = threading.Lock()
_last_recorded_throughput_sample = None
_MODEL_DISCOVERY_TIMEOUT_SECONDS = float(os.environ.get("DASHBOARD_MODEL_DISCOVERY_TIMEOUT", "15.0"))
_MIN_MODEL_CONTEXT = 1024
_MAX_MODEL_CONTEXT = 9007199254740991
_AGENT_MODEL_STATUS_CACHE_TTL_SECONDS = float(
    os.environ.get("DASHBOARD_AGENT_MODEL_STATUS_CACHE_TTL", "0.5")
)
_STALE_TERMINAL_DOWNLOAD_STATUS_SECONDS = float(
    os.environ.get("DASHBOARD_STALE_TERMINAL_DOWNLOAD_STATUS_SECONDS", "1800")
)
_STALE_ACTIVE_BOOTSTRAP_STATUS_SECONDS = float(
    os.environ.get("DASHBOARD_STALE_ACTIVE_BOOTSTRAP_STATUS_SECONDS", "900")
)
_ACTIVE_BOOTSTRAP_STATUSES = {"starting", "downloading", "verifying", "swapping"}
_agent_model_status_cache_lock = threading.Lock()
_agent_model_status_cache_at = 0.0
_agent_model_status_cache_value: Optional[dict] = None
_GPU_VRAM_EXCEPTIONS = (
    ImportError,
    FileNotFoundError,
    OSError,
    KeyError,
    AttributeError,
)


def _model_lifecycle_from_agent_status(status: Optional[dict]) -> Optional[dict[str, Any]]:
    if not isinstance(status, dict):
        return None
    operation = status.get("activeOperation")
    active = bool(status.get("lifecycleActive") or operation)
    if not active or not isinstance(operation, str) or not operation:
        return None
    target = status.get("activeTarget")
    model_id = status.get("activeModelId") or target
    return {
        "active": True,
        "operation": operation,
        "target": target,
        "modelId": model_id,
    }


def _annotate_model_lifecycle(payload: dict[str, Any], lifecycle: Optional[dict[str, Any]]) -> None:
    if not lifecycle:
        return
    payload["modelLifecycle"] = lifecycle
    if lifecycle.get("operation") != "model_activation":
        return
    target = lifecycle.get("modelId") or lifecycle.get("target")
    if not target:
        return
    for model in payload.get("models") or []:
        if isinstance(model, dict) and current_model_matches(model, str(target), str(target)):
            model["modelOperation"] = lifecycle
            return


def _configured_ods_mode() -> str:
    """Return the current persisted mode without treating process env as config."""
    try:
        return normalize_ods_mode(read_env_file_value("ODS_MODE", INSTALL_DIR, raise_on_error=True))
    except PermissionError:
        # The private .env can belong to a different host UID than this API.
        # Ask its owner-side agent for only the persisted mode, never secrets
        # or the process environment. Read freshly before every activation.
        try:
            snapshot = request_agent_json("GET", "/v1/model/config", timeout=5)
        except AgentClientError:
            return "unknown"
        mode = snapshot.get("configuredMode") if isinstance(snapshot, dict) else None
        return mode if isinstance(mode, str) and mode in {"local", "cloud", "hybrid"} else "unknown"
    except (OSError, UnicodeError):
        return "unknown"


def _model_activation_mode_denial(
    effective_mode: str,
    configured_mode: str,
    llm_backend: str = "",
) -> dict[str, str] | None:
    """Describe why this runtime cannot safely perform a local model swap."""
    effective_mode = normalize_ods_mode(effective_mode)
    configured_mode = normalize_ods_mode(configured_mode)
    normalized_backend = str(llm_backend or "").strip().lower()
    if normalized_backend == "external":
        code = "external_llm_managed"
        reason = "external_backend_selected"
        message = (
            "Local model activation is unavailable while ODS uses your own "
            "model server. Change the model in that server, or rerun the "
            "installer with an ODS-managed backend to activate downloaded models."
        )
    elif normalized_backend == "vllm":
        # The vLLM engine serves one installer-set snapshot; activation swaps
        # llama-server GGUFs (the host agent refuses it as well).
        code = "vllm_runtime_installer_managed"
        reason = "vllm_backend_selected"
        message = (
            "This install runs its model on the vLLM engine, and models cannot "
            "be switched here. To use another model, run the ODS installer "
            "again and choose it."
        )
    elif "unknown" in {effective_mode, configured_mode}:
        code = "ods_mode_unknown"
        reason = "mode_unknown"
        message = (
            "Local model activation is unavailable because the effective or "
            "configured ODS mode is unknown."
        )
    elif effective_mode != configured_mode:
        code = "ods_mode_mismatch"
        reason = "mode_mismatch"
        message = (
            f"Local model activation is unavailable because effective mode "
            f"'{effective_mode}' does not match configured mode '{configured_mode}'."
        )
    elif effective_mode not in LOCAL_MODEL_MODES:
        code = "local_mode_required"
        reason = "effective_mode_not_local"
        message = (
            f"Local model activation is unavailable while effective ODS mode "
            f"is '{effective_mode}'."
        )
    else:
        return None

    return {
        "error": "local_mode_required",
        "code": code,
        "reason": reason,
        "message": message,
        "effectiveMode": effective_mode,
        "configuredMode": configured_mode,
        "llmBackend": normalized_backend or "unknown",
    }

try:
    import pynvml
except ImportError:
    pynvml = None
else:
    _GPU_VRAM_EXCEPTIONS = _GPU_VRAM_EXCEPTIONS + (pynvml.NVMLError,)


def _local_model_name_from_gguf(gguf_file: str) -> str:
    name = re.sub(r"[^A-Za-z0-9._-]+", "-", Path(gguf_file).stem).strip("-._")
    return name or "local-gguf"


def _imported_library_path() -> Path:
    return Path(DATA_DIR) / "model-imports.json"


def _read_model_records(
    path: Path,
    *,
    required: bool,
    strict: bool = False,
) -> list[dict[str, Any]]:
    if not path.exists():
        if required:
            logger.warning("Model library not found: %s", path)
        return []
    try:
        payload = json.loads(path.read_text(encoding="utf-8"))
    except (json.JSONDecodeError, OSError, UnicodeError) as exc:
        logger.warning("Failed to load model library %s: %s", path, exc)
        if strict:
            raise HTTPException(
                status_code=409,
                detail="The Hugging Face import registry is unreadable; it was not overwritten",
            ) from exc
        return []
    records = payload.get("models") if isinstance(payload, dict) else None
    if not isinstance(records, list):
        logger.warning("Model library %s does not contain a models array", path)
        if strict:
            raise HTTPException(
                status_code=409,
                detail="The Hugging Face import registry is malformed; it was not overwritten",
            )
        return []
    if strict and not all(isinstance(record, dict) for record in records):
        raise HTTPException(
            status_code=409,
            detail="The Hugging Face import registry contains invalid records; it was not overwritten",
        )
    return [record for record in records if isinstance(record, dict)]


def _load_library() -> list[dict]:
    """Load the curated catalog plus separately persisted Hub imports."""
    curated = _read_model_records(_LIBRARY_PATH, required=True)
    imported = _read_model_records(_imported_library_path(), required=False)
    seen_ids = {str(model.get("id") or "") for model in curated}
    seen_files = {str(model.get("gguf_file") or "").lower() for model in curated}
    merged = list(curated)
    for model in imported:
        model_id = str(model.get("id") or "")
        filename = str(model.get("gguf_file") or "").lower()
        if (
            model.get("source") != "huggingface"
            or not model_id
            or not filename
            or model_id in seen_ids
            or filename in seen_files
        ):
            continue
        merged.append(model)
        seen_ids.add(model_id)
        seen_files.add(filename)
    return merged


def _write_imported_library(records: list[dict[str, Any]]) -> None:
    """Atomically persist the Hub import allowlist on the shared data volume."""
    target = _imported_library_path()
    target.parent.mkdir(parents=True, exist_ok=True)
    if target.is_symlink():
        raise HTTPException(status_code=409, detail="Refusing to replace a symlinked model import registry")
    content = json.dumps(
        {"version": 1, "models": records},
        indent=2,
        sort_keys=True,
    ) + "\n"
    descriptor, temporary_name = tempfile.mkstemp(
        dir=target.parent,
        prefix=f".{target.name}.",
        suffix=".tmp",
    )
    temporary = Path(temporary_name)
    try:
        with os.fdopen(descriptor, "w", encoding="utf-8", newline="\n") as handle:
            handle.write(content)
            handle.flush()
            os.fsync(handle.fileno())
        os.chmod(temporary, 0o644)
        os.replace(temporary, target)
    finally:
        temporary.unlink(missing_ok=True)


def _scan_downloaded_models() -> dict[str, int]:
    """Scan data/models/ for downloaded GGUF files. Returns {filename: size_bytes}."""
    downloaded = {}
    for name, path in _installed_model_paths().items():
        try:
            downloaded[name] = path.stat().st_size
        except OSError:
            continue
    return downloaded


def _is_final_gguf_file(path: Path) -> bool:
    try:
        return path.is_file() and path.name.lower().endswith(".gguf") and path.stat().st_size > 0
    except OSError:
        return False


def _read_active_model() -> Optional[str]:
    """Read the currently active GGUF_FILE from .env."""
    if not _ENV_PATH.exists():
        return None
    try:
        for line in _ENV_PATH.read_text(encoding="utf-8").splitlines():
            if line.startswith("GGUF_FILE="):
                return parse_env_value(line.split("=", 1)[1])
    except PermissionError:
        # The install owner's private .env (an owner other than UID 1000):
        # its host agent reads the active file for this API.
        return (owner_model_config() or {}).get("ggufFile")
    except OSError:
        pass
    return None


def _strip_llm_api_suffix(base_url: str) -> str:
    base = base_url.strip().rstrip("/")
    for suffix in ("/api/v1", "/v1", "/api"):
        if base.endswith(suffix):
            return base[: -len(suffix)].rstrip("/")
    return base


def _configured_llm_base_url(host: str, port: int) -> str:
    # LiteLLM's LLM_API_URL is an alias gateway, not the physical runtime.
    # Model identity probes must follow the server that serves the model.
    if LLM_BACKEND == "external":
        value = read_env_value("EXTERNAL_LLM_CONTAINER_URL", INSTALL_DIR)
        if value:
            return _strip_llm_api_suffix(value)
    if read_env_value("AMD_INFERENCE_LOCATION", INSTALL_DIR).strip().lower() == "host":
        # A Windows-hosted llama-server, as containers reach it (the legacy
        # key name is read for one release).
        for key in ("NATIVE_LLM_CONTAINER_BASE_URL", "LEMONADE_CONTAINER_BASE_URL"):
            value = read_env_value(key, INSTALL_DIR)
            if value:
                return _strip_llm_api_suffix(value)
    for key in ("LLM_URL", "LLM_API_URL", "OLLAMA_URL"):
        value = read_env_value(key, INSTALL_DIR)
        if value and "litellm" not in value.lower():
            return _strip_llm_api_suffix(value)
    return f"http://{host}:{port}"


def _model_name_tokens(value: str | None) -> set[str]:
    if not value:
        return set()
    token = Path(str(value).strip()).name
    if not token:
        return set()
    lower = token.lower()
    tokens = {lower}
    # Retired Lemonade ids stay matchable for one release, so a persisted
    # receipt or performance row still names its GGUF.
    for prefix in ("extra.", "user."):
        if lower.startswith(prefix):
            tokens.add(lower[len(prefix):])
    for candidate in tuple(tokens):
        if candidate.endswith(".gguf"):
            tokens.add(candidate[:-5])
    return tokens


def _catalog_model_tokens(model: dict) -> set[str]:
    tokens: set[str] = set()
    for key in ("id", "gguf_file", "llm_model_name"):
        tokens.update(_model_name_tokens(model.get(key)))
    gguf_file = model.get("gguf_file")
    if gguf_file:
        tokens.update(_model_name_tokens(f"extra.{gguf_file}"))
    return tokens


def _fetch_loaded_model_sync() -> str | None:
    service = SERVICES.get("llama-server", {})
    host = service.get("host", "llama-server")
    port = int(service.get("port", 8080))
    loop = asyncio.new_event_loop()
    try:
        return loop.run_until_complete(_fetch_llama_loaded_model(host, port))
    except (httpx.HTTPError, OSError, RuntimeError, ValueError):
        return None
    finally:
        loop.close()


def _read_activation_receipt() -> dict:
    path = Path(DATA_DIR) / "model-activation-receipt.json"
    try:
        value = json.loads(path.read_text(encoding="utf-8"))
    except (OSError, json.JSONDecodeError, TypeError):
        return {}
    return value if isinstance(value, dict) else {}


def _activation_receipt_matches(
    model_id: str | None,
    model: dict,
    loaded_model: str | None,
) -> bool:
    if not model_id or not loaded_model:
        return False
    receipt = _read_activation_receipt()
    if (
        receipt.get("schema") != "ods.model-activation-receipt.v1"
        or receipt.get("status") != "complete"
        or receipt.get("modelId") != model_id
        or receipt.get("contextVerified") is False
    ):
        return False

    gguf_file = str(model.get("gguf_file") or model.get("gguf") or "").strip()
    if not gguf_file or str(receipt.get("ggufFile") or "").casefold() != gguf_file.casefold():
        return False
    if not isinstance(receipt.get("consumers"), dict):
        return False

    runtime_tokens = _model_name_tokens(receipt.get("runtimeModelId"))
    loaded_tokens = _model_name_tokens(loaded_model)
    return bool(runtime_tokens and loaded_tokens and runtime_tokens & loaded_tokens)


def _verified_activation_context(loaded_model: str | None) -> int | None:
    if not loaded_model:
        return None
    receipt = _read_activation_receipt()
    if (
        receipt.get("schema") != "ods.model-activation-receipt.v1"
        or receipt.get("status") != "complete"
        or receipt.get("contextVerified") is not True
        or not (_model_name_tokens(receipt.get("runtimeModelId")) & _model_name_tokens(loaded_model))
    ):
        return None
    try:
        context = int(receipt.get("contextLength") or 0)
    except (TypeError, ValueError):
        return None
    return context if context > 0 else None


def _configured_model_identity_matches(model: dict) -> bool:
    gguf_file = model.get("gguf_file")
    if not gguf_file:
        return False
    if _read_active_model() != gguf_file:
        return False
    configured_llm = (
        read_env_file_value("LLM_MODEL", INSTALL_DIR)
        or read_env_value("LLM_MODEL", INSTALL_DIR)
    )
    if not (_model_name_tokens(configured_llm) & _catalog_model_tokens(model)):
        return False
    if not (Path(DATA_DIR) / "models" / gguf_file).exists():
        return False
    return True


def _already_active_model(model_id: str, model: dict) -> tuple[bool, str | None]:
    # Fetch the live backend identity even when the bind-mounted .env identity
    # is stale. Model activation replaces .env atomically on the host, so a
    # long-running container with a single-file bind mount can retain the old
    # inode until it is recreated.
    loaded_model = _fetch_loaded_model_sync()
    if not _configured_model_identity_matches(model):
        return False, loaded_model

    if _model_name_tokens(loaded_model) & _catalog_model_tokens(model):
        # llama-server serves exactly one model, proven by the activation
        # receipt; a chat probe here would only turn an idempotent Run click
        # into a long wait.
        if _activation_receipt_matches(model_id, model, loaded_model):
            return True, loaded_model
    return False, loaded_model


def _requested_activation_context(
    body: dict[str, Any] | None,
) -> int | None:
    if body is None or "context_length" not in body:
        return None
    value = body.get("context_length")
    if isinstance(value, bool) or not isinstance(value, int):
        raise HTTPException(
            status_code=400,
            detail="context_length must be an integer",
        )
    if not _MIN_MODEL_CONTEXT <= value <= _MAX_MODEL_CONTEXT:
        raise HTTPException(
            status_code=400,
            detail=f"context_length must be a safe integer of at least {_MIN_MODEL_CONTEXT}",
        )
    return value


def _policy_activation_context(model_id: str, preferred_context: int | None = None) -> int | None:
    """Context a switch to ``model_id`` serves: the installer's policy.

    performance_oracle.activation_context_plan runs the installer's selector
    code for this one model on this hardware (the Hermes floor when it fits,
    otherwise the largest context that does), so a dashboard switch, the
    model list and a fresh install agree. None keeps the host agent's own
    default (unknown hardware, an import outside the catalog, or a model that
    fits at no context).
    """
    entry = _find_normalized_model(model_id)
    if entry is None:
        return None
    try:
        gpu = get_gpu_info()
    except _GPU_VRAM_EXCEPTIONS as exc:
        logger.debug("GPU detection failed while planning activation context: %s", exc)
        gpu = None
    plan = activation_context_plan(entry, INSTALL_DIR, gpu, preferred_context=preferred_context)
    if not plan or not plan.get("fits"):
        return None
    try:
        context = int(plan.get("context_length") or 0)
    except (TypeError, ValueError):
        return None
    return context if _MIN_MODEL_CONTEXT <= context <= _MAX_MODEL_CONTEXT else None


def _recommended_model_context(model: dict) -> int | None:
    """Installer-recorded context when ``model`` is the installer's pick."""
    identity = {
        str(value).casefold()
        for value in (
            read_env_file_value("MODEL_RECOMMENDED_GGUF", INSTALL_DIR),
            read_env_file_value("MODEL_RECOMMENDED_MODEL", INSTALL_DIR),
        )
        if value
    }
    if not identity & {
        str(value).casefold()
        for value in (model.get("gguf_file"), model.get("llm_model_name"), model.get("id"))
        if value
    }:
        return None
    try:
        context = int(str(read_env_file_value("MODEL_RECOMMENDED_CONTEXT", INSTALL_DIR) or "").strip())
    except (TypeError, ValueError):
        return None
    return context if context > 0 else None


def _configured_context_length() -> int | None:
    keys = ("CTX_SIZE", "MAX_CONTEXT")
    for reader in (read_env_file_value, read_env_value):
        for key in keys:
            value = reader(key, INSTALL_DIR)
            try:
                parsed = int(str(value).strip())
            except (TypeError, ValueError):
                continue
            if parsed > 0:
                return parsed
    return None


async def _await_or_default(coro, default, label: str, timeout_seconds: float = 2.0):
    try:
        return await asyncio.wait_for(coro, timeout=timeout_seconds)
    except (asyncio.TimeoutError, httpx.HTTPError, OSError, RuntimeError, KeyError) as exc:
        logger.debug("%s unavailable: %s", label, exc)
        return default


def _get_gpu_vram() -> Optional[ModelLibraryGpu]:
    """Get GPU VRAM info for model compatibility gating."""
    try:
        from gpu import get_gpu_info
        gpu = get_gpu_info()
        if gpu is None:
            return None
        total_gb = gpu.memory_total_mb / 1024
        used_gb = gpu.memory_used_mb / 1024
        return ModelLibraryGpu(
            vramTotal=round(total_gb, 1),
            vramUsed=round(used_gb, 1),
            vramFree=round(total_gb - used_gb, 1),
        )
    except _GPU_VRAM_EXCEPTIONS as exc:
        logger.warning("GPU VRAM detection failed: %s", exc)
        return None


def _format_size(size_mb: int) -> str:
    """Format size in MB to a human-readable string."""
    if size_mb >= 1024:
        return f"{size_mb / 1024:.1f} GB"
    return f"{size_mb} MB"


def _hf_token() -> str:
    return str(
        read_env_file_value("HF_TOKEN", INSTALL_DIR)
        or read_env_value("HF_TOKEN", INSTALL_DIR)
        or ""
    ).strip()


def _hf_cache_identity() -> str:
    """Partition metadata caches without retaining or exposing the token."""
    token = _hf_token()
    return hashlib.sha256(token.encode("utf-8")).hexdigest() if token else "public"


def _hf_headers() -> dict[str, str]:
    headers = {"User-Agent": "ODS-dashboard/2.5 model-library"}
    token = _hf_token()
    if token:
        headers["Authorization"] = f"Bearer {token}"
    return headers


def _hf_cache_get(
    cache_key: tuple[str, str, int, str],
) -> tuple[float, dict[str, Any]] | None:
    """Read and refresh one bounded least-recently-used search entry."""
    with _HF_SEARCH_CACHE_LOCK:
        cached = _HF_SEARCH_CACHE.pop(cache_key, None)
        if cached is not None:
            _HF_SEARCH_CACHE[cache_key] = cached
        return cached


def _hf_cache_put(
    cache_key: tuple[str, str, int, str],
    response: dict[str, Any],
) -> None:
    """Bound arbitrary search/token combinations while retaining stale fallback."""
    with _HF_SEARCH_CACHE_LOCK:
        _HF_SEARCH_CACHE.pop(cache_key, None)
        _HF_SEARCH_CACHE[cache_key] = (time.monotonic(), response)
        while len(_HF_SEARCH_CACHE) > _HF_SEARCH_CACHE_MAX_ENTRIES:
            _HF_SEARCH_CACHE.pop(next(iter(_HF_SEARCH_CACHE)))


def _hf_avatar_cache_get(cache_key: tuple[str, str]) -> tuple[float, str | None] | None:
    with _HF_AVATAR_CACHE_LOCK:
        cached = _HF_AVATAR_CACHE.pop(cache_key, None)
        if cached is not None:
            _HF_AVATAR_CACHE[cache_key] = cached
        return cached


def _hf_avatar_cache_put(cache_key: tuple[str, str], avatar_url: str | None) -> None:
    with _HF_AVATAR_CACHE_LOCK:
        _HF_AVATAR_CACHE.pop(cache_key, None)
        _HF_AVATAR_CACHE[cache_key] = (time.monotonic(), avatar_url)
        while len(_HF_AVATAR_CACHE) > _HF_AVATAR_CACHE_MAX_ENTRIES:
            _HF_AVATAR_CACHE.pop(next(iter(_HF_AVATAR_CACHE)))


def _hf_trusted_avatar_url(value: Any) -> str | None:
    """Normalize Hub avatar metadata without permitting an arbitrary redirect."""
    if not isinstance(value, str) or not value.strip():
        return None
    avatar_url = urljoin(f"{_HF_API_BASE}/", value.strip())
    parsed = urlsplit(avatar_url)
    if (
        parsed.scheme != "https"
        or parsed.hostname not in _HF_AVATAR_HOSTS
        or parsed.port is not None
        or parsed.username is not None
        or parsed.password is not None
    ):
        return None
    return avatar_url


async def _hf_author_avatar_url(author: str) -> str | None:
    """Resolve an author's uploaded Hub avatar, caching positive and negative results."""
    if not _HF_AUTHOR_RE.fullmatch(author):
        return None
    cache_key = (author.lower(), _hf_cache_identity())
    now = time.monotonic()
    cached = _hf_avatar_cache_get(cache_key)
    if cached and now - cached[0] < _HF_AVATAR_CACHE_TTL_SECONDS:
        return cached[1]

    for account_type in ("organizations", "users"):
        try:
            payload, _headers = await _hf_get_json(
                f"/api/{account_type}/{quote(author, safe='')}/overview",
            )
        except HTTPException as exc:
            if exc.status_code == 404:
                continue
            logger.info("Hugging Face avatar lookup failed for %s: %s", author, exc.detail)
            return None
        if isinstance(payload, dict):
            avatar_url = _hf_trusted_avatar_url(payload.get("avatarUrl"))
            _hf_avatar_cache_put(cache_key, avatar_url)
            return avatar_url

    _hf_avatar_cache_put(cache_key, None)
    return None


def _hf_license(payload: dict[str, Any]) -> str | None:
    card_data = payload.get("cardData")
    if isinstance(card_data, dict) and card_data.get("license"):
        return str(card_data["license"])
    raw_tags = payload.get("tags")
    for tag in raw_tags if isinstance(raw_tags, list) else []:
        if isinstance(tag, str) and tag.startswith("license:"):
            return tag.split(":", 1)[1]
    return None


def _hf_quantization(filename: str) -> str | None:
    match = _HF_QUANT_RE.search(Path(filename).name)
    return match.group("quant").upper() if match else None


def _hf_context_length(payload: dict[str, Any]) -> tuple[int | None, str]:
    raw_gguf = payload.get("gguf")
    gguf = raw_gguf if isinstance(raw_gguf, dict) else {}
    value = gguf.get("context_length")
    if (
        isinstance(value, int)
        and not isinstance(value, bool)
        and 512 <= value <= _MAX_MODEL_CONTEXT
    ):
        return value, "gguf_metadata"

    raw_config = payload.get("config")
    config = raw_config if isinstance(raw_config, dict) else {}
    raw_text_config = config.get("text_config")
    text_config = raw_text_config if isinstance(raw_text_config, dict) else {}
    for source in (text_config, config):
        for key in (
            "max_position_embeddings",
            "max_sequence_length",
            "model_max_length",
            "n_ctx",
            "seq_length",
        ):
            value = source.get(key)
            # Generic model configs sometimes publish enormous tokenizer
            # sentinels instead of a usable runtime window. Keep that defensive
            # ceiling for config fallbacks; the GGUF summary above is authoritative.
            if (
                isinstance(value, int)
                and not isinstance(value, bool)
                and 512 <= value <= 4_194_304
            ):
                return value, "hub_config"
    return None, "unavailable"


def _hf_file_metadata(sibling: Any) -> tuple[int | None, str | None]:
    if not isinstance(sibling, dict):
        return None, None
    lfs = sibling.get("lfs") if isinstance(sibling.get("lfs"), dict) else {}
    raw_size = lfs.get("size") or sibling.get("size")
    raw_sha = lfs.get("sha256")
    try:
        size = int(raw_size)
    except (TypeError, ValueError):
        size = None
    sha = str(raw_sha or "").lower()
    if not re.fullmatch(r"[0-9a-f]{64}", sha):
        sha = None
    return size if size and size > 0 else None, sha


def _hf_nonnegative_int(value: Any) -> int:
    try:
        return max(0, int(value or 0))
    except (TypeError, ValueError, OverflowError):
        return 0


# Speculative-decoding heads published beside a model's weights: EAGLE-3
# drafts (ggml-org's "eagle3-*.gguf"), multi-token-prediction heads
# (unsloth's "MTP/mtp-*.gguf") and DFlash drafts. None runs on its own, and
# read as the repository's header one misdescribes the model: gpt-oss-20b
# was refused as "eagle3" on b9014 and gemma-4-E4B shown without a chat
# template (fleet, 2026-10-09).
_HF_SPECULATIVE_HEAD_RE = re.compile(r"(?:^|[/._-])(?:eagle3|mtp|dflash)(?:[/._-]|$)")


def _hf_supported_gguf_filename(filename: str) -> bool:
    basename = Path(filename).name.lower()
    if not basename.endswith(".gguf"):
        return False
    if _HF_SPECULATIVE_HEAD_RE.search(filename.lower()):
        return False
    # imatrix: llama.cpp importance-matrix calibration data that quantizers
    # publish next to the weights (bartowski's "*-imatrix.gguf"); not a model.
    unsupported_markers = ("mmproj", "projector", "adapter", "lora", "tokenizer", "imatrix")
    return not any(marker in basename for marker in unsupported_markers)


def _hf_llm_runtime_compatibility(payload: dict[str, Any]) -> tuple[bool, str | None]:
    pipeline = str(payload.get("pipeline_tag") or "text-generation").lower()
    repo_id = str(payload.get("id") or payload.get("modelId") or "").lower()
    raw_tags = payload.get("tags")
    tags = {
        str(tag).lower()
        for tag in raw_tags
    } if isinstance(raw_tags, list) else set()
    unsupported_pipelines = {
        "automatic-speech-recognition",
        "audio-classification",
        "feature-extraction",
        "image-classification",
        "image-to-image",
        "sentence-similarity",
        "text-ranking",
        "text-to-image",
        "text-to-speech",
        "zero-shot-image-classification",
    }
    if pipeline in unsupported_pipelines:
        return False, f"{pipeline.replace('-', ' ').title()} requires a dedicated ODS runtime"
    unsupported_tags = {
        "automatic-speech-recognition",
        "feature-extraction",
        "sentence-transformers",
        "text-ranking",
        "text-to-image",
        "text-to-speech",
    }
    if tags & unsupported_tags:
        return False, "This repository targets a non-LLM runtime"
    if re.search(r"(^|[-_/])(embed|embedding|asr|whisper|tts)([-_/]|$)", repo_id):
        return False, "This repository targets a non-LLM runtime"
    return True, None


def _hf_artifact_id(filenames: list[str]) -> str:
    return hashlib.sha256("\n".join(filenames).encode("utf-8")).hexdigest()[:20]


def _hf_gguf_artifacts(payload: dict[str, Any]) -> list[dict[str, Any]]:
    siblings = payload.get("siblings") if isinstance(payload.get("siblings"), list) else []
    ggufs = []
    for sibling in siblings:
        filename = str(sibling.get("rfilename") or "") if isinstance(sibling, dict) else ""
        if not _hf_supported_gguf_filename(filename):
            continue
        size, sha = _hf_file_metadata(sibling)
        if size is None or sha is None:
            continue
        ggufs.append({"filename": filename, "sizeBytes": size, "sha256": sha})

    split_groups: dict[tuple[str, str, int], list[tuple[int, dict[str, Any]]]] = {}
    singles: list[dict[str, Any]] = []
    for artifact in ggufs:
        match = _HF_SPLIT_GGUF_RE.match(Path(artifact["filename"]).name)
        if not match:
            singles.append(artifact)
            continue
        parent = Path(artifact["filename"]).parent.as_posix()
        key = (parent, match.group("prefix"), int(match.group("total")))
        split_groups.setdefault(key, []).append((int(match.group("part")), artifact))

    groups: list[dict[str, Any]] = []
    for artifact in singles:
        groups.append({
            "id": _hf_artifact_id([artifact["filename"]]),
            "label": Path(artifact["filename"]).name,
            "quantization": _hf_quantization(artifact["filename"]),
            "sizeBytes": artifact["sizeBytes"],
            "files": [artifact],
            "split": False,
        })
    for (parent, prefix, total), parts in split_groups.items():
        parts.sort(key=lambda item: item[0])
        if len(parts) != total or [number for number, _ in parts] != list(range(1, total + 1)):
            continue
        files = [artifact for _, artifact in parts]
        groups.append({
            "id": _hf_artifact_id([artifact["filename"] for artifact in files]),
            "label": f"{'' if parent == '.' else parent + '/'}{prefix} ({total} parts)",
            "quantization": _hf_quantization(prefix),
            "sizeBytes": sum(artifact["sizeBytes"] for artifact in files),
            "files": files,
            "split": True,
        })
    groups.sort(key=lambda item: (item["sizeBytes"], item["label"].lower()))
    return groups


_HF_PROJECTOR_PRECISION_ORDER = ("F16", "BF16", "F32", "Q8_0")


def _hf_gguf_projectors(payload: dict[str, Any]) -> list[dict[str, Any]]:
    """Vision projector files (mmproj) a repository publishes next to its weights."""
    siblings = payload.get("siblings") if isinstance(payload.get("siblings"), list) else []
    projectors = []
    for sibling in siblings:
        filename = str(sibling.get("rfilename") or "") if isinstance(sibling, dict) else ""
        basename = Path(filename).name.lower()
        if not basename.endswith(".gguf") or "mmproj" not in basename:
            continue
        size, sha = _hf_file_metadata(sibling)
        if size is None or sha is None:
            continue
        projectors.append({
            "id": _hf_artifact_id([filename]),
            "label": Path(filename).name,
            "filename": filename,
            "sizeBytes": size,
            "sha256": sha,
            "precision": _hf_quantization(filename),
        })
    projectors.sort(key=lambda item: str(item["label"]).lower())
    return projectors


def _hf_default_projector(projectors: list[dict[str, Any]]) -> dict[str, Any] | None:
    """F16, then BF16 (then F32, Q8_0), else the only one: llama.cpp -hf's closest-name pick."""
    for precision in _HF_PROJECTOR_PRECISION_ORDER:
        matches = [item for item in projectors if str(item.get("precision") or "").upper() == precision]
        if matches:
            return matches[0]
    return projectors[0] if len(projectors) == 1 else None


def _hf_search_item(payload: dict[str, Any]) -> dict[str, Any] | None:
    repo_id = str(payload.get("id") or payload.get("modelId") or "")
    if not _HF_REPO_RE.fullmatch(repo_id):
        return None
    raw_siblings = payload.get("siblings")
    sibling_names = [
        str(item.get("rfilename") or "")
        for item in raw_siblings
        if isinstance(item, dict)
    ] if isinstance(raw_siblings, list) else []
    raw_tags = payload.get("tags")
    tags = [str(tag) for tag in raw_tags[:12]] if isinstance(raw_tags, list) else []
    gguf_count = sum(_hf_supported_gguf_filename(name) for name in sibling_names)
    runtime_compatible, runtime_reason = _hf_llm_runtime_compatibility(payload)
    return {
        "id": repo_id,
        "author": repo_id.split("/", 1)[0],
        "name": repo_id.split("/", 1)[1],
        "downloads": _hf_nonnegative_int(payload.get("downloads")),
        "likes": _hf_nonnegative_int(payload.get("likes")),
        "lastModified": payload.get("lastModified"),
        "pipelineTag": payload.get("pipeline_tag") or "text-generation",
        "gated": bool(payload.get("gated")),
        "private": bool(payload.get("private")),
        "license": _hf_license(payload),
        "ggufFileCount": gguf_count,
        "runtimeCompatible": runtime_compatible,
        "runtimeReason": runtime_reason,
        "tags": tags,
        "url": f"{_HF_API_BASE}/{repo_id}",
    }


async def _hf_get_json(
    path: str,
    *,
    params: dict[str, Any] | None = None,
    timeout_seconds: float = 20.0,
    connect_timeout_seconds: float = 8.0,
) -> tuple[Any, httpx.Headers]:
    timeout = httpx.Timeout(
        timeout_seconds,
        connect=min(connect_timeout_seconds, timeout_seconds),
    )
    response: httpx.Response | None = None
    last_error: HTTPException | None = None
    for attempt in range(2):
        try:
            async with httpx.AsyncClient(
                timeout=timeout,
                follow_redirects=False,
            ) as client:
                response = await client.get(
                    f"{_HF_API_BASE}{path}",
                    params=params,
                    headers=_hf_headers(),
                )
            break
        except httpx.TimeoutException as exc:
            last_error = HTTPException(status_code=504, detail="Hugging Face did not respond in time")
            last_error.__cause__ = exc
        except httpx.HTTPError as exc:
            last_error = HTTPException(status_code=502, detail=f"Hugging Face request failed: {exc}")
            last_error.__cause__ = exc
        if attempt == 0:
            await asyncio.sleep(0.25)
    if response is None:
        assert last_error is not None
        raise last_error
    if response.status_code in {401, 403}:
        raise HTTPException(
            status_code=403,
            detail="This Hugging Face repository requires an accepted license and a valid HF_TOKEN",
        )
    if response.status_code == 404:
        raise HTTPException(status_code=404, detail="Hugging Face repository not found")
    if response.status_code == 429:
        raise HTTPException(status_code=429, detail="Hugging Face rate limit reached; retry later or configure HF_TOKEN")
    if response.status_code >= 400:
        raise HTTPException(status_code=502, detail=f"Hugging Face returned HTTP {response.status_code}")
    try:
        return response.json(), response.headers
    except ValueError as exc:
        raise HTTPException(status_code=502, detail="Hugging Face returned invalid JSON") from exc


async def _hf_repo_details(repo_id: str) -> dict[str, Any]:
    if not _HF_REPO_RE.fullmatch(repo_id):
        raise HTTPException(status_code=400, detail="Invalid Hugging Face repository id")
    payload, _headers = await _hf_get_json(
        f"/api/models/{quote(repo_id, safe='/')}",
        params={
            "blobs": "true",
            "expand": [
                "gguf",
                "sha",
                "downloads",
                "likes",
                "lastModified",
                "pipeline_tag",
                "gated",
                "private",
                "tags",
                "cardData",
            ],
        },
        timeout_seconds=60.0,
        connect_timeout_seconds=20.0,
    )
    if not isinstance(payload, dict):
        raise HTTPException(status_code=502, detail="Hugging Face returned an invalid repository record")
    artifacts = _hf_gguf_artifacts(payload)
    projectors = _hf_gguf_projectors(payload)
    vision_unavailable = await asyncio.to_thread(_projector_unavailable_reason) if projectors else None
    default_projector = None if vision_unavailable else _hf_default_projector(projectors)
    imported_by_artifact = {
        str(record.get("source_artifact_id") or ""): record
        for record in _read_model_records(_imported_library_path(), required=False)
        if record.get("source") == "huggingface"
        and record.get("source_repo") == repo_id
        and record.get("source_revision") == str(payload.get("sha") or "")
    }
    for artifact in artifacts:
        imported = imported_by_artifact.get(artifact["id"])
        artifact["importedModelId"] = imported.get("id") if imported else None
        if imported:
            filenames = [
                str(part.get("file") or "")
                for part in imported.get("gguf_parts") or []
                if isinstance(part, dict)
            ] or [str(imported.get("gguf_file") or "")]
            artifact["installed"] = bool(filenames) and all(
                _installed_model_path(filename) is not None
                for filename in filenames
            )
        else:
            artifact["installed"] = False
    context_length, context_source = _hf_context_length(payload)
    runtime_compatible, runtime_reason = _hf_llm_runtime_compatibility(payload)
    raw_gguf = payload.get("gguf") if isinstance(payload.get("gguf"), dict) else {}
    raw_tags = payload.get("tags")
    return {
        "id": repo_id,
        "sha": str(payload.get("sha") or ""),
        # The Hub summary describes one file only; the per-file header read
        # in the preflight is authoritative.
        "ggufArchitecture": raw_gguf.get("architecture") if isinstance(raw_gguf.get("architecture"), str) else None,
        "tags": [str(tag) for tag in raw_tags] if isinstance(raw_tags, list) else [],
        "downloads": _hf_nonnegative_int(payload.get("downloads")),
        "likes": _hf_nonnegative_int(payload.get("likes")),
        "lastModified": payload.get("lastModified"),
        "pipelineTag": payload.get("pipeline_tag") or "text-generation",
        "gated": bool(payload.get("gated")),
        "private": bool(payload.get("private")),
        "license": _hf_license(payload),
        "contextLength": context_length,
        "contextSource": context_source,
        "runtimeCompatible": runtime_compatible,
        "runtimeReason": runtime_reason,
        "artifacts": artifacts,
        # Vision projectors (WP2): one is imported with the chosen weights by default.
        "projectors": projectors,
        "defaultProjectorId": default_projector["id"] if default_projector else None,
        "visionUnavailableReason": vision_unavailable,
        "authenticated": bool(_hf_token()),
        "url": f"{_HF_API_BASE}/{repo_id}",
    }


_ARCHITECTURES_PATH = Path(INSTALL_DIR) / "config" / "llama-cpp-architectures.json"


def _preflight_header_artifact(artifacts: list[dict[str, Any]]) -> dict[str, Any] | None:
    """Default to one small artifact; its header speaks only for that artifact."""
    candidates = [artifact for artifact in artifacts if artifact.get("files")]
    if not candidates:
        return None
    return min(candidates, key=lambda artifact: (_hf_artifact_size(artifact), str(artifact.get("label") or "")))


def _hf_artifact_binding(details: dict[str, Any], artifact: dict[str, Any] | None) -> dict[str, Any]:
    return {
        "repoId": details["id"],
        "revision": details["sha"],
        "artifactId": artifact.get("id") if artifact else None,
        "file": artifact["files"][0]["filename"] if artifact and artifact.get("files") else None,
    }


def _hf_artifact_context(details: dict[str, Any]) -> tuple[int | None, str]:
    # Hub metadata describes one file, without identifying which artifact.
    if len(details.get("artifacts") or []) != 1:
        return None, "unavailable"
    return details.get("contextLength"), details.get("contextSource") or "unavailable"


def _hf_artifact_size(artifact: dict[str, Any]) -> int:
    """Total bytes of one artifact (all split parts)."""
    declared = artifact.get("sizeBytes")
    if isinstance(declared, int) and not isinstance(declared, bool) and declared > 0:
        return declared
    return sum(
        int(item.get("sizeBytes") or 0)
        for item in artifact.get("files") or []
        if isinstance(item, dict)
    )


async def _hf_preflight_gate(
    details: dict[str, Any],
    artifact: dict[str, Any] | None = None,
    *,
    read_header: bool = True,
) -> dict[str, Any]:
    """The header read and the two hard refusals, shared by preflight and import.

    The import passes ``read_header=False`` and uses only a header the
    preflight already read, so a slow Hub never delays an import; without one
    the gate can use Hub hints only for a repository with one artifact.
    """
    header = None
    header_status: dict[str, Any] = {"status": "unavailable", "reason": "no_artifact"}
    if artifact is None:
        artifact = _preflight_header_artifact(details.get("artifacts") or [])
    source = artifact["files"][0] if artifact and artifact.get("files") else None
    if source is not None and not read_header:
        header = hf_gguf_header.cached_gguf_header(details["id"], details["sha"], source["filename"])
        header_status = (
            {"status": "read", "file": source["filename"], "bytesRead": header.get("bytes_read")}
            if header else {"status": "unavailable", "file": source["filename"], "reason": "not_read"}
        )
    elif source is not None:
        try:
            header = await hf_gguf_header.fetch_gguf_header(
                details["id"],
                details["sha"],
                source["filename"],
                expected_size=source.get("sizeBytes"),
                token=_hf_token(),
            )
            header_status = {
                "status": "read",
                "file": source["filename"],
                "bytesRead": header.get("bytes_read"),
            }
        except hf_gguf_header.HeaderUnavailable as exc:
            header_status = {
                "status": "unavailable",
                "file": source["filename"],
                "reason": exc.code,
                "message": exc.message,
            }
    policy = model_preflight.load_runtime_policy(_ARCHITECTURES_PATH)
    runtime_key, build = model_preflight.effective_build(
        policy,
        GPU_BACKEND,
        windows_hosted=_windows_hosted_runtime(),
    )
    single_artifact = len(details.get("artifacts") or []) == 1
    header_architecture = (header or {}).get("architecture")
    summary_architecture = details.get("ggufArchitecture") if single_artifact else None
    architecture = (
        header_architecture
        if isinstance(header_architecture, str) and header_architecture not in {"", "unknown"}
        else summary_architecture
    )
    kind = model_preflight.model_kind(
        header,
        details.get("pipelineTag") if single_artifact else None,
        details.get("tags") if single_artifact else None,
    )
    supported = (
        model_preflight.architecture_supported(policy, build, architecture)
        if kind == "chat" else None
    )
    # The Hub serves a gated repository's files only with a token whose
    # account accepted the license: refuse when no token is set, or when the
    # header read with the token was itself refused as gated.
    gated = bool(details.get("gated")) and (
        not _hf_token() or header_status.get("reason") == "gated"
    )
    refusal = model_preflight.refusal(kind, supported, architecture, build, gated=gated)
    if (refusal is None and kind == "chat" and model_preflight.nextn_layers(header)
            and model_preflight.nextn_supported(policy, build, architecture) is False):
        refusal = model_preflight.nextn_refusal(build)
    return {
        "artifact": _hf_artifact_binding(details, artifact),
        "header": header,
        "headerStatus": header_status,
        "policy": policy,
        "architecture": architecture,
        "runtime": {"key": runtime_key, "build": build, "architectureSupported": supported},
        "modelKind": kind,
        "refusal": refusal,
    }


# Resolving a Windows-managed model store from WSL asks Windows first (about 5 s
# on the fleet's Strix Halo host), the same cost /v1/model/management carries.
_MODEL_STORAGE_TIMEOUT_SECONDS = 20


async def _model_storage_status() -> dict[str, Any] | None:
    """Free space where downloads land, from the host agent; None when unknown."""
    try:
        value = await asyncio.to_thread(
            request_agent_json, "GET", "/v1/model/storage", timeout=_MODEL_STORAGE_TIMEOUT_SECONDS,
        )
    except AgentClientError:
        return None
    if not isinstance(value, dict):
        return None
    status = {key: value.get(key) for key in ("freeBytes", "totalBytes", "marginBytes")}
    # WSL only: whether the Windows drive under the virtual disk was counted.
    status.update({key: value[key] for key in ("windowsDrive", "windowsDriveChecked") if key in value})
    return status


def _hf_default_context(declared: int | None) -> int:
    """The context an import starts at: min(declared, 32K), or 8K when unknown."""
    if isinstance(declared, int) and 512 <= declared <= _MAX_MODEL_CONTEXT:
        return min(declared, 32768)
    return 8192


def _hf_artifact_fit(
    artifact: dict[str, Any],
    layout: dict[str, Any],
    declared_context: int | None,
    gpu_info: Any,
    extra_bytes: int = 0,
) -> dict[str, Any]:
    """Will this quantization fit, and at what context, on this machine?

    ``extra_bytes`` is the vision projector loaded with the weights (WP2).
    """
    estimate = "architecture" if "recurrent_state_bytes" in layout else "rough"
    if gpu_info is None:
        return {"status": "unknown", "estimate": estimate}
    size = _hf_artifact_size(artifact) + max(int(extra_bytes or 0), 0)
    model = {
        "id": f"hf-preflight-{artifact['id']}",
        "gguf_file": artifact["files"][0]["filename"],
        "size_bytes": size,
        "size_mb": round(size / (1024 ** 2), 2),
        "context_length": _hf_default_context(declared_context),
        **layout,
    }
    plan = planned_model_context(model, gpu_info)
    if not plan["fits"]:
        status = "too_large"
    elif plan["meets_min_context"]:
        status = "fits"
    else:
        status = "fits_short_context"
    return {
        "status": status,
        "contextLength": plan["context_length"],
        "requiredGb": plan["required_gb"],
        "capacityGb": plan["capacity_gb"],
        "estimate": estimate,
    }


def _hf_artifact_tensors(gate: dict[str, Any], artifact: dict[str, Any]) -> dict[str, Any]:
    """WP1.6: can this host's llama.cpp read the artifact's tensor types?

    The header read for the repository speaks for its own file only; every
    other quantization is judged by a label that names a ggml type outright.
    """
    header = gate.get("header") or {}
    files = artifact.get("files") or []
    same_file = bool(files) and files[0].get("filename") == (gate.get("headerStatus") or {}).get("file")
    check = model_preflight.artifact_tensor_check(
        gate.get("policy"),
        gate["runtime"]["build"],
        artifact.get("quantization"),
        header.get("tensor_types") if same_file else None,
    )
    if check["status"] == "unsupported":
        check["refusal"] = model_preflight.tensor_refusal(gate["runtime"]["build"], check["unknown"])
    return check


async def _hf_preflight(
    details: dict[str, Any], artifact: dict[str, Any] | None = None,
) -> dict[str, Any]:
    # The Hub header read and the host's own answers are independent: wait for
    # the slowest, not their sum.
    gate, gpu_info, storage = await asyncio.gather(
        _hf_preflight_gate(details, artifact),
        asyncio.to_thread(get_gpu_info),
        _model_storage_status(),
    )
    header = gate["header"]
    layout = model_preflight.memory_fields(header)
    fallback_context, fallback_source = _hf_artifact_context(details)
    declared_context = layout.get("max_context_length") or fallback_context
    context_source = "gguf_header" if layout.get("max_context_length") else fallback_source
    projector = next((item for item in details.get("projectors") or []
                      if item["id"] == details.get("defaultProjectorId")), None)
    projector_bytes = int(projector["sizeBytes"]) if projector else 0
    artifacts = {}
    for artifact in details.get("artifacts") or []:
        artifact_gate = gate if gate["artifact"] == _hf_artifact_binding(details, artifact) else await _hf_preflight_gate(
            details, artifact, read_header=False,
        )
        artifact_layout = model_preflight.memory_fields(artifact_gate["header"])
        artifact_context = artifact_layout.get("max_context_length") or fallback_context
        # Access denial applies to the repository; model evidence does not.
        refusal = gate["refusal"] if (gate["refusal"] or {}).get("code") == "gated" else artifact_gate["refusal"]
        weights = 0 if artifact.get("installed") else _hf_artifact_size(artifact)
        artifacts[artifact["id"]] = {
            "header": artifact_gate["headerStatus"],
            "architecture": artifact_gate["architecture"],
            "template": model_preflight.template_signals(artifact_gate["header"]),
            "contextLength": artifact_context,
            "contextSource": "gguf_header" if artifact_layout.get("max_context_length") else fallback_source,
            "refusal": refusal,
            "fit": _hf_artifact_fit(artifact, artifact_layout, artifact_context, gpu_info, projector_bytes),
            "disk": model_preflight.disk_status(weights + projector_bytes if weights else 0, storage),
            # An import with Include vision unticked downloads the weights alone.
            "diskWithoutProjector": model_preflight.disk_status(weights, storage),
            "tensors": _hf_artifact_tensors(artifact_gate, artifact),
        }
    return {
        "id": details["id"],
        "sha": details["sha"],
        "artifactId": gate["artifact"]["artifactId"],
        "header": gate["headerStatus"],
        "architecture": gate["architecture"],
        "runtime": gate["runtime"],
        "modelKind": gate["modelKind"],
        "contextLength": declared_context,
        "contextSource": context_source,
        "template": model_preflight.template_signals(header),
        "storage": storage,
        "artifacts": artifacts,
        "projector": {key: projector[key] for key in ("id", "label", "sizeBytes", "precision")} if projector else None,
        "refusal": gate["refusal"],
    }


@router.get("/api/models/huggingface/preflight/{repo_id:path}")
async def huggingface_repository_preflight(
    repo_id: str,
    artifactId: str | None = None,
    api_key: str = Depends(verify_api_key),
):
    """What ODS can tell about a repository's GGUFs before downloading one."""
    details = await _hf_repo_details(repo_id)
    artifact = None
    if artifactId is not None:
        artifact = next((item for item in details["artifacts"] if item["id"] == artifactId), None)
        if artifact is None:
            raise HTTPException(status_code=409, detail="The selected GGUF artifact is no longer available at this revision")
    return await _hf_preflight(details, artifact)


def _hf_local_filename(repo_id: str, remote_filename: str, revision: str) -> str:
    repo_slug = re.sub(r"[^A-Za-z0-9._-]+", "-", repo_id).strip("-._")
    basename = Path(remote_filename).name
    split = _HF_SPLIT_GGUF_RE.fullmatch(basename)
    stem = split.group("prefix") if split else Path(basename).stem
    stem = re.sub(r"[^A-Za-z0-9._-]+", "-", stem).strip("-._")
    suffix = ".gguf"
    identity = remote_filename
    if split:
        # llama.cpp derives sibling paths from a shared prefix followed by
        # -00001-of-00002.gguf. Keep the digest common to the complete set and
        # before that suffix, including when long names must be shortened.
        suffix = f"-{split.group('part')}-of-{split.group('total')}.gguf"
        identity = Path(remote_filename).with_name(f"{split.group('prefix')}-of-{split.group('total')}.gguf").as_posix()
    digest = hashlib.sha256(
        f"{repo_id}\n{revision}\n{identity}".encode("utf-8")
    ).hexdigest()[:8]
    filename = f"hf-{repo_slug}-{stem}-{digest}{suffix}"
    if len(filename) > 220:
        filename = f"hf-{repo_slug[:60]}-{stem[:120]}-{digest}{suffix}"
    return filename


def _hf_projector_fields(details: dict[str, Any], projector: dict[str, Any]) -> dict[str, Any]:
    """The import record's vision projector: downloaded, verified and deleted with the weights."""
    repo_id, revision = details["id"], details["sha"]
    return {
        "mmproj_file": _hf_local_filename(repo_id, projector["filename"], revision),
        "mmproj_url": f"{_HF_API_BASE}/{quote(repo_id, safe='/')}/resolve/{revision}/{quote(projector['filename'], safe='/')}",
        "mmproj_sha256": projector["sha256"],
        "mmproj_size_bytes": projector["sizeBytes"],
        "mmproj_source_file": projector["filename"],
    }


def _hf_requested_projector(details: dict[str, Any], body: dict[str, Any]) -> dict[str, Any] | None:
    """The projector an import brings: the chosen one, else the default, unless vision is off."""
    if body.get("includeVision") is False or not details.get("projectors") or not details.get("defaultProjectorId"):
        return None
    wanted = str(body.get("projectorId") or details.get("defaultProjectorId") or "")
    projector = next((item for item in details["projectors"] if item["id"] == wanted), None)
    if body.get("projectorId") and projector is None:
        raise HTTPException(status_code=409, detail="The selected vision projector is no longer available at this revision")
    return projector


def _refuse_vllm_entry(model: dict[str, Any], action: str) -> None:
    """409 for a vLLM catalog entry: the installer downloads and runs it.

    Its snapshot is a model folder, not a GGUF the model lifecycle can
    download, switch to or delete (scripts/download-hf-manifest.py and
    LLM_BACKEND=vllm).
    """
    if not is_vllm_entry(model):
        return
    raise HTTPException(status_code=409, detail={
        "error": f"{model.get('name') or model.get('id')} runs only on the vLLM engine. {action}",
        "code": "vllm_model_installer_managed",
        "requestedModelId": model.get("id"),
    })


def _hf_download_payload(record: dict[str, Any]) -> dict[str, Any]:
    """The host-agent download request for a model record, projector included."""
    payload: dict[str, Any] = {
        "gguf_file": record["gguf_file"],
        "gguf_url": record.get("gguf_url", ""),
        "gguf_sha256": record.get("gguf_sha256", ""),
    }
    if record.get("gguf_parts"):
        payload["gguf_parts"] = record["gguf_parts"]
    if record.get("mmproj_file"):
        payload["mmproj"] = {
            "file": record["mmproj_file"],
            "url": record.get("mmproj_url", ""),
            "sha256": record.get("mmproj_sha256", ""),
        }
    return payload


def _hf_import_record(
    details: dict[str, Any],
    artifact: dict[str, Any],
    *,
    gate: dict[str, Any] | None = None,
    runtime_override: dict[str, Any] | None = None,
    projector: dict[str, Any] | None = None,
) -> dict[str, Any]:
    bound = (gate or {}).get("artifact") == _hf_artifact_binding(details, artifact)
    header = (gate or {}).get("header") if bound else None
    layout = model_preflight.memory_fields(header)
    context_length, context_source = _hf_artifact_context(details)
    details = {**details, "contextLength": context_length, "contextSource": context_source}
    if layout.get("max_context_length"):
        # Only the selected artifact's own header can supply its layout/context.
        details = {**details, "contextLength": layout["max_context_length"], "contextSource": "gguf_header"}
    record = _hf_import_record_base(details, artifact)
    if header:
        record.update({key: value for key, value in layout.items() if key != "max_context_length"})
        record["architecture"] = (gate or {}).get("architecture")
        record["template_signals"] = model_preflight.template_signals(header)
    if runtime_override and bound:
        record["runtime_override"] = runtime_override
    if projector:
        record.update(_hf_projector_fields(details, projector))
    return record


def _hf_import_record_base(details: dict[str, Any], artifact: dict[str, Any]) -> dict[str, Any]:
    repo_id = details["id"]
    revision = details["sha"]
    if not re.fullmatch(r"[0-9a-fA-F]{40,64}", revision):
        raise HTTPException(status_code=502, detail="Hugging Face did not provide an immutable repository revision")
    remote_files = artifact["files"]
    if not remote_files:
        raise HTTPException(status_code=422, detail="The selected artifact contains no files")
    local_files = [
        _hf_local_filename(repo_id, item["filename"], revision)
        for item in remote_files
    ]
    if len(set(local_files)) != len(local_files):
        raise HTTPException(status_code=409, detail="The selected artifact contains colliding local filenames")
    parts = []
    for remote, local_filename in zip(remote_files, local_files):
        parts.append({
            "file": local_filename,
            "url": f"{_HF_API_BASE}/{quote(repo_id, safe='/')}/resolve/{revision}/{quote(remote['filename'], safe='/')}",
            "sha256": remote["sha256"],
            "size_bytes": remote["sizeBytes"],
            "source_file": remote["filename"],
        })
    total_size = sum(item["sizeBytes"] for item in remote_files)
    quantization = artifact.get("quantization") or "unknown"
    digest = hashlib.sha256(
        f"{repo_id}\n{revision}\n{artifact['id']}".encode("utf-8")
    ).hexdigest()[:12]
    model_id = f"hf-{re.sub(r'[^a-z0-9]+', '-', repo_id.lower()).strip('-')[:72]}-{digest}"
    declared_context = details.get("contextLength")
    try:
        max_context_length = int(declared_context) if declared_context is not None else 0
    except (TypeError, ValueError, OverflowError):
        max_context_length = 0
    context_limit_known = 512 <= max_context_length <= _MAX_MODEL_CONTEXT
    context_length = min(max_context_length, 32768) if context_limit_known else 8192
    size_gb = total_size / (1024 ** 3)
    record: dict[str, Any] = {
        "id": model_id,
        "name": f"{repo_id.split('/', 1)[1]} · {quantization}",
        "family": repo_id.split("/", 1)[1].split("-", 1)[0].lower(),
        "gguf_file": local_files[0],
        "gguf_url": parts[0]["url"],
        "gguf_sha256": parts[0]["sha256"],
        "size_bytes": total_size,
        "size_mb": round(total_size / (1024 ** 2), 2),
        "vram_required_gb": round(size_gb + min(max(size_gb * 0.18, 0.5), 3.5), 2),
        "context_length": context_length,
        "max_context_length": max_context_length if context_limit_known else None,
        "context_limit_known": context_limit_known,
        "quantization": quantization,
        "specialty": "Community GGUF",
        "description": f"Imported from Hugging Face repository {repo_id}. Not validated by ODS.",
        "llm_model_name": model_id,
        "source": "huggingface",
        "source_repo": repo_id,
        "source_revision": revision,
        "source_artifact_id": artifact["id"],
        "source_url": details["url"],
        "license": details.get("license"),
        "context_source": details.get("contextSource"),
        "app_compatibility": {
            "openai_chat": {
                "status": "unknown",
                "label": "Community model not validated",
                "reason": "Run a local benchmark and compatibility check after download.",
            },
            "hermes_talk": {
                "status": "unknown",
                "label": "ODS Talk not validated",
                "reason": "Community Hugging Face imports are not part of the ODS compatibility matrix.",
            },
            "agent_viability": {
                "status": "unknown",
                "label": "Agent viability not validated",
                "reason": "Tool calling and instruction behavior vary by community model.",
            },
        },
        "imported_at": datetime.now(timezone.utc).isoformat(),
    }
    if len(parts) > 1:
        record["gguf_parts"] = parts
    else:
        record["size_bytes"] = parts[0]["size_bytes"]
    return record


@router.get("/api/models/huggingface/search")
async def search_huggingface_models(
    q: str = Query(default="", max_length=100),
    sort: str = Query(default="downloads"),
    limit: int = Query(default=20, ge=1, le=30),
    api_key: str = Depends(verify_api_key),
):
    """Search public/authenticated Hub metadata without exposing the token."""
    query = q.strip()
    sort_key = sort if sort in {"downloads", "likes", "lastModified"} else "downloads"
    cache_key = (query.lower(), sort_key, limit, _hf_cache_identity())
    now = time.monotonic()
    cached = _hf_cache_get(cache_key)
    if cached and now - cached[0] < _HF_SEARCH_CACHE_TTL_SECONDS:
        return cached[1]
    params: dict[str, Any] = {
        "filter": "gguf",
        "sort": sort_key,
        "direction": -1,
        "limit": limit,
        "full": "true",
    }
    if query:
        params["search"] = query
    try:
        payload = None
        if _HF_REPO_RE.fullmatch(query):
            # A pasted repository ID does not need the Hub's search index.
            # The metadata endpoint can remain healthy during search outages.
            try:
                repository, _headers = await _hf_get_json(
                    f"/api/models/{quote(query, safe='/')}",
                    timeout_seconds=6.0,
                    connect_timeout_seconds=3.0,
                )
            except HTTPException as exc:
                if exc.status_code != 404:
                    raise
                # An owner/name prefix can still be a useful fuzzy search.
            else:
                if not isinstance(repository, dict) or str(
                    repository.get("id") or repository.get("modelId") or ""
                ).lower() != query.lower():
                    raise HTTPException(status_code=502, detail="Hugging Face returned an unexpected repository")
                item = _hf_search_item(repository)
                payload = [repository] if item and item["ggufFileCount"] else []
        if payload is None:
            payload, _headers = await _hf_get_json("/api/models", params=params)
    except HTTPException as exc:
        if cached and exc.status_code in {429, 502, 504}:
            return {**cached[1], "stale": True}
        raise
    if not isinstance(payload, list):
        raise HTTPException(status_code=502, detail="Hugging Face returned an invalid search result")
    models = [item for raw in payload if isinstance(raw, dict) if (item := _hf_search_item(raw))]
    response = {
        "models": models,
        "query": query,
        "sort": sort_key,
        "authenticated": bool(_hf_token()),
        "source": "huggingface",
    }
    _hf_cache_put(cache_key, response)
    return response


@router.get("/api/models/huggingface/authors/{author}/avatar")
async def huggingface_author_avatar(
    author: str,
    api_key: str = Depends(verify_api_key),
):
    """Redirect to the uploaded avatar declared by an official Hub profile."""
    if not _HF_AUTHOR_RE.fullmatch(author):
        raise HTTPException(status_code=400, detail="Invalid Hugging Face author")
    avatar_url = await _hf_author_avatar_url(author)
    if avatar_url is None:
        raise HTTPException(status_code=404, detail="Hugging Face author has no uploaded avatar")
    return RedirectResponse(
        avatar_url,
        status_code=307,
        headers={"Cache-Control": "public, max-age=3600"},
    )


@router.get("/api/models/huggingface/repositories/{repo_id:path}")
async def huggingface_repository_details(
    repo_id: str,
    api_key: str = Depends(verify_api_key),
):
    """Return integrity-qualified GGUF choices for one Hub repository."""
    return await _hf_repo_details(repo_id)


async def _prepare_huggingface_import(body: dict[str, Any]):
    """Prepare metadata without submitting a download to the host."""
    repo_id = str(body.get("repoId") or "").strip()
    artifact_id = str(body.get("artifactId") or "").strip()
    if not _HF_REPO_RE.fullmatch(repo_id) or not re.fullmatch(r"[0-9a-f]{20}", artifact_id):
        raise HTTPException(status_code=400, detail="repoId and artifactId are required")
    details = await _hf_repo_details(repo_id)
    if "revision" in body and body["revision"] != details.get("sha"):
        raise HTTPException(status_code=409, detail="The repository revision changed. Reopen it and check the selected file before importing.")
    if not details["runtimeCompatible"]:
        raise HTTPException(status_code=422, detail=details["runtimeReason"])
    artifact = next((item for item in details["artifacts"] if item["id"] == artifact_id), None)
    if artifact is None:
        raise HTTPException(status_code=409, detail="The selected GGUF artifact is no longer available at this revision")
    if (details.get("private") or details.get("gated")) and not _hf_token():
        raise HTTPException(
            status_code=403,
            detail="Private or gated repositories require HF_TOKEN",
        )
    gate = await _hf_preflight_gate(details, artifact, read_header=False)
    refusal = gate["refusal"]
    runtime_override = None
    if refusal is not None:
        if not (refusal["overridable"] and body.get("allowUnsupportedRuntime") is True):
            raise HTTPException(status_code=422, detail={
                "code": refusal["code"],
                "message": refusal["message"],
                "overridable": refusal["overridable"],
            })
        runtime_override = {
            "code": refusal["code"],
            "architecture": gate["architecture"],
            "build": gate["runtime"]["build"],
            "acknowledgedAt": datetime.now(timezone.utc).isoformat(),
        }
        logger.warning(
            "Hugging Face import of %s acknowledged %s for architecture %s on llama.cpp %s",
            repo_id, refusal["code"], gate["architecture"], gate["runtime"]["build"],
        )
    tensors = _hf_artifact_tensors(gate, artifact)
    if tensors["status"] == "unsupported":
        tensor_refusal = tensors["refusal"]
        if body.get("allowUnsupportedRuntime") is not True:
            raise HTTPException(status_code=422, detail=tensor_refusal)
        runtime_override = {
            **(runtime_override or {}),
            "code": tensor_refusal["code"] if runtime_override is None else runtime_override["code"],
            "tensorTypes": tensors["unknown"],
            "build": gate["runtime"]["build"],
            "acknowledgedAt": datetime.now(timezone.utc).isoformat(),
        }
        logger.warning(
            "Hugging Face import of %s (%s) acknowledged tensor types %s that llama.cpp %s cannot read",
            repo_id, artifact["label"], tensors["unknown"], gate["runtime"]["build"],
        )
    projector = _hf_requested_projector(details, body)
    record = _hf_import_record(details, artifact, gate=gate, runtime_override=runtime_override, projector=projector)

    bootstrap_conflict = _bootstrap_upgrade_download_conflict()
    if bootstrap_conflict is not None:
        raise HTTPException(
            status_code=409,
            detail={**bootstrap_conflict, "requestedModelId": record["id"]},
        )

    with _IMPORTED_MODELS_LOCK:
        records = _read_model_records(
            _imported_library_path(),
            required=False,
            strict=True,
        )
        previous = next(
            (
                item for item in records
                if item.get("id") == record["id"]
                and item.get("source_revision") == record["source_revision"]
                and item.get("source_artifact_id") == record["source_artifact_id"]
            ),
            None,
        )
        if previous and previous.get("imported_at"):
            record["imported_at"] = previous["imported_at"]
        record_filename = str(record.get("gguf_file") or "").lower()
        retained = [
            item for item in records
            if item.get("id") != record["id"]
            and str(item.get("gguf_file") or "").lower() != record_filename
        ]
        retained.append(record)
        _write_imported_library(retained)

    return details, artifact, record


@router.post("/api/models/huggingface/import")
async def import_huggingface_model(
    body: dict[str, Any] = Body(...),
    api_key: str = Depends(verify_api_key),
):
    """Pin, register, and start one integrity-qualified Hub GGUF download."""
    # A failure before dispatch is a definitive refusal, even if its status
    # is 500. A transport failure after dispatch remains uncertain: never
    # encourage the UI to replay a potentially accepted host operation.
    try:
        details, artifact, record = await _prepare_huggingface_import(body)
    except HTTPException as exc:
        exc.headers = {**(exc.headers or {}), "X-ODS-Import-Started": "false"}
        raise
    except Exception as exc:
        logger.exception("Hugging Face import preparation failed before dispatch")
        raise HTTPException(
            status_code=500,
            detail="Could not prepare the import. No download was started; you can retry.",
            headers={"X-ODS-Import-Started": "false"},
        ) from exc

    payload = _hf_download_payload(record)
    try:
        result = await asyncio.to_thread(_request_agent_download, payload)
    except HTTPException as exc:
        busy = exc.status_code == 409 and isinstance(exc.detail, dict) \
            and exc.detail.get("code") == "model_lifecycle_busy"
        if exc.status_code == 507 or busy:
            # The host refused before starting: a definitive, retryable refusal.
            exc.headers = {**(exc.headers or {}), "X-ODS-Import-Started": "false"}
        raise
    return {
        **result,
        "modelId": record["id"],
        "repoId": details["id"],
        "artifact": artifact["label"],
        "revision": details["sha"],
    }


def _newly_measured_tps(metrics: dict, loaded_model: str | None) -> float:
    # Sticky Dashboard values are historical evidence, not a fresh performance
    # sample for the model catalogue/current context on every polling request.
    if (metrics.get("throughput_state") != "measured"
            or metrics.get("throughput_model") != loaded_model
            # Runtime live intervals can aggregate concurrent slots; do not
            # turn them into single-request model performance/benchmark data.
            or metrics.get("throughput_mode") == "live_output_interval"):
        return 0.0
    return float(metrics.get("tokens_per_second") or 0)


_RUNTIME_ERROR_LIMIT = 1024
_START_REFUSALS = ("memory_refused", "gpu_busy")


def _model_management_proof() -> dict:
    """The host agent's management proof, with whether its launcher loads a vision projector."""
    if not _windows_hosted_runtime():
        return {"managed": False, "canActivate": False, "canUnload": False, "running": False, "vision": False}
    try:
        value = request_agent_json("GET", "/v1/model/management", timeout=20)
        if not isinstance(value, dict) or any(type(value.get(key)) is not bool for key in (
            "managed", "canActivate", "canUnload", "running"
        )):
            raise ValueError("Invalid model management response")
        if (not value['managed'] and any(value[key] for key in ('canActivate', 'canUnload', 'running'))
                or value['canActivate'] and not value['running']):
            raise ValueError("Inconsistent model management response")
        result = {key: value[key] for key in ("managed", "canActivate", "canUnload", "running")}
        result["vision"] = value["managed"] and value.get("vision") is True
        if isinstance(value.get('reason'), str):
            result['reason'] = value['reason'][:500]
        # Why a managed Windows model is not running (the launcher's own
        # error), and a memory refusal its Resume overrides ("Start anyway").
        error = value.get("runtimeError")
        if value["managed"] and not value["running"] and isinstance(error, str) and error.strip():
            result["runtimeError"] = error[:_RUNTIME_ERROR_LIMIT]
            if value.get("startRefusal") in _START_REFUSALS:
                result["startRefusal"] = value["startRefusal"]
        return result
    except (AgentClientError, ValueError):
        # A failed proof is unknown, not evidence of an independently managed service.
        return {"managed": None, "canActivate": False, "canUnload": False, "running": False, "vision": False,
                "reason": "Runtime management could not be verified"}


def _model_management() -> dict:
    """Project capability evidence; a network topology flag grants no control."""
    proof = _model_management_proof()
    if not _windows_hosted_runtime():
        return {key: proof[key] for key in ("managed", "canActivate", "canUnload", "running")}
    return {key: value for key, value in proof.items() if key != "vision"}


# The ownership proof runs a controller on the Windows host and takes seconds,
# more under load: on an RTX Spark it took 3.6-5.4 s during a Portal build and
# was the slowest part of every GET /api/models. The catalog reuses the last
# conclusive proof while the live runtime agrees with it and refreshes it in
# the background. A runtime change made here forgets it, an owned lifecycle
# operation always gets a fresh one, and the host proves ownership again
# before any control acts.
_MODEL_MANAGEMENT_REFRESH_SECONDS = 15.0
_MODEL_MANAGEMENT_MAX_AGE_SECONDS = 6 * 3600.0
_model_management_lock = threading.Lock()
_model_management_remembered: tuple[float, dict] | None = None
_model_management_generation = 0
_model_management_proof_future: asyncio.Future | None = None
_model_management_missed = False


def _forget_model_management() -> None:
    """Drop the remembered proof, and any proof still running from before a runtime change."""
    global _model_management_remembered, _model_management_generation, _model_management_proof_future
    global _model_management_missed
    with _model_management_lock:
        _model_management_remembered = None
        _model_management_generation += 1
        _model_management_proof_future = None
        _model_management_missed = False


async def _prove_model_management(generation: int) -> tuple[int, dict]:
    global _model_management_remembered, _model_management_missed
    value = await asyncio.to_thread(_model_management)
    with _model_management_lock:
        if generation == _model_management_generation:
            if value.get("managed") is not None:
                _model_management_remembered = (time.monotonic(), dict(value))
                _model_management_missed = False
            elif _model_management_missed:
                # Two unverified proofs in a row: the host keeps failing, so the
                # catalog stops showing the last good one. A single interop
                # timeout leaves it in place.
                _model_management_remembered = None
            else:
                _model_management_missed = True
    return generation, value


def _log_model_management_crash(done: asyncio.Future) -> None:
    # A background refresh has no reader, so its crash is logged here.
    if not done.cancelled() and done.exception() is not None:
        logger.error("Model management proof crashed", exc_info=done.exception())


def _model_management_in_flight() -> asyncio.Future:
    """The proof already running, or a new one; concurrent catalog reads share it."""
    global _model_management_proof_future
    with _model_management_lock:
        future = _model_management_proof_future
        if future is None or future.done():
            future = asyncio.ensure_future(_prove_model_management(_model_management_generation))
            future.add_done_callback(_log_model_management_crash)
            _model_management_proof_future = future
    return future


def _remembered_model_management() -> tuple[float, dict] | None:
    with _model_management_lock:
        remembered = _model_management_remembered
    if remembered is None or time.monotonic() - remembered[0] > _MODEL_MANAGEMENT_MAX_AGE_SECONDS:
        return None
    return remembered


async def _fresh_model_management(future: asyncio.Future) -> dict:
    """The proof ``future`` yields, taken again if a runtime change landed meanwhile."""
    generation, value = await asyncio.shield(future)
    if generation != _model_management_generation:
        _generation, value = await asyncio.shield(_model_management_in_flight())
    return dict(value)


async def _catalog_model_management(prefetch: asyncio.Future | None, agent_status: object,
                                    loaded_model: str | None) -> dict:
    """A fresh proof, or the remembered one when the live runtime agrees with it."""
    if prefetch is not None:
        return await _fresh_model_management(prefetch)
    remembered = _remembered_model_management()
    lifecycle = isinstance(agent_status, dict) and (
        agent_status.get("lifecycleActive") is True or agent_status.get("modelTransactionPending") is True)
    if remembered is not None and not lifecycle:
        taken_at, value = remembered
        # An owned runtime runs exactly when its model answers, so a crash or
        # a stop made elsewhere never keeps the earlier controls on screen.
        if value.get("managed") is not True or value.get("running") is (loaded_model is not None):
            if time.monotonic() - taken_at > _MODEL_MANAGEMENT_REFRESH_SECONDS:
                _model_management_in_flight()
            return dict(value)
    return await _fresh_model_management(_model_management_in_flight())


# Docker Desktop installs whose host agent launches llama-server.exe on
# Windows itself (the host agent's _WINDOWS_NATIVE_RUNTIME_MODES); that
# launch takes no projector.
_LEGACY_WINDOWS_NATIVE_MODES = frozenset({"windows-native-llama-server", "windows-llama-server-fallback"})


def _projector_unavailable_reason() -> str | None:
    """Why this runtime cannot load a vision projector, or None when it can."""
    mode = read_live_env_values(("AMD_INFERENCE_RUNTIME_MODE",)).get("AMD_INFERENCE_RUNTIME_MODE")
    if str(mode or "").strip().casefold() in _LEGACY_WINDOWS_NATIVE_MODES:
        return "The Windows llama.cpp runtime of this installation does not load vision files, so the model imports without one"
    if not _windows_hosted_runtime():
        return None
    management = _model_management_proof()
    if management.get("vision") is True:
        return None
    if management.get("managed") is True:
        return ("This Windows model runtime was set up before vision support, so the model imports "
                "without its vision file. Run Windows setup again to add vision")
    return "ODS could not confirm that the Windows model runtime loads vision files, so the model imports without one"


async def _discover_loaded_model() -> str | None:
    """Try both identity sources within the catalog's one discovery budget."""
    loaded_model = await get_loaded_model()
    if loaded_model:
        return loaded_model
    service = SERVICES.get("llama-server", {})
    return await _fetch_llama_loaded_model(
        service.get("host", "llama-server"), int(service.get("port", 8080)),
    )


@router.get("/api/models", response_model=ModelLibraryResponse)
async def list_models(api_key: str = Depends(verify_api_key)):
    """List model catalog entries with source-labelled performance metadata."""
    global _last_recorded_throughput_sample
    host_runtime = _windows_hosted_runtime()
    # With no proof to reuse, the slow ownership proof runs alongside these reads.
    management_prefetch = (_model_management_in_flight()
                           if host_runtime and _remembered_model_management() is None else None)
    gpu_info, loaded_model, agent_status = await asyncio.gather(
        asyncio.to_thread(get_gpu_info),
        _await_or_default(
            _discover_loaded_model(),
            None,
            "loaded model discovery",
            timeout_seconds=_MODEL_DISCOVERY_TIMEOUT_SECONDS,
        ),
        asyncio.to_thread(_get_agent_model_status),
    )
    metrics, context_size = await asyncio.gather(
        _await_or_default(
            get_llama_metrics(model_hint=loaded_model),
            {"tokens_per_second": 0, "lifetime_tokens": 0},
            "llama metrics",
        ),
        _await_or_default(
            get_llama_context_size(model_hint=loaded_model),
            None,
            "llama context",
        ),
    )
    context_size = context_size or _verified_activation_context(loaded_model)
    live_tps = _newly_measured_tps(metrics, loaded_model)
    payload = await asyncio.to_thread(
        build_models_payload,
        gpu_info,
        loaded_model,
        live_tps,
        INSTALL_DIR,
        DATA_DIR,
        context_size,
        catalog=_load_library(),
        downloaded_files_override=_installed_model_paths(),
    )
    _annotate_model_lifecycle(
        payload,
        _model_lifecycle_from_agent_status(agent_status),
    )
    payload["modelActivation"] = model_activation_status(agent_status)
    payload["modelRecoveryPending"] = isinstance(agent_status, dict) and agent_status.get("modelTransactionPending") is True
    loaded_entry = next((m for m in payload["models"] if m["status"] == "loaded"), None) or {}
    sample_key = (loaded_model, metrics.get("throughput_sampled_at"))
    if (gpu_info and loaded_model and live_tps > 0
            and sample_key[1] is not None and sample_key != _last_recorded_throughput_sample
            and loaded_entry.get("metadata", {}).get("source") != "runtime"):
        _last_recorded_throughput_sample = sample_key
        signature = build_sample_signature(
            loaded_entry or {"id": loaded_model, "gguf": _read_active_model()},
            gpu_info,
            context_size,
            INSTALL_DIR,
            _installed_model_path(loaded_entry["gguf"]) if loaded_entry.get("gguf") else None,
        )
        await asyncio.to_thread(
            record_model_performance,
            loaded_model,
            gpu_info.name,
            gpu_info.gpu_backend,
            live_tps,
            model_id=signature.get("model_id"),
            gguf=signature.get("gguf"),
            quantization=signature.get("quantization"),
            architecture=signature.get("architecture"),
            context_length=signature.get("context_length"),
            decode_read_mb=signature.get("decode_read_mb"),
            vram_total_mb=signature.get("vram_total_mb"),
            os_name=signature.get("os"),
            flags=signature.get("flags"),
        )
    payload["odsMode"] = ODS_MODE_EFFECTIVE
    payload["configuredMode"] = await asyncio.to_thread(_configured_ods_mode)
    payload["llmBackend"] = LLM_BACKEND or "unknown"
    if LLM_BACKEND == "external":
        # API mode: name the model and the API's host so the page can say
        # what serves chat. The installer refuses URLs with credentials.
        payload["externalModel"] = read_env_value("EXTERNAL_LLM_MODEL", INSTALL_DIR).strip() or None
        payload["externalHost"] = _external_api_host(read_env_value("EXTERNAL_LLM_URL", INSTALL_DIR))
    payload["hostRuntime"] = host_runtime
    if host_runtime:
        payload["modelManagement"] = await _catalog_model_management(
            management_prefetch, agent_status, loaded_model)
    # A held transition does not establish which route currently serves chat.
    payload["activeRuntime"] = (
        None if payload.get("modelRecoveryPending") or (payload.get("modelLifecycle") or {}).get("active")
        else active_runtime_projection(agent_status)
    )
    payload["activationReadyModel"] = (
        payload.get("currentModel")
        if loaded_entry
        and _activation_receipt_matches(payload.get("currentModel"), loaded_entry, loaded_model)
        else None
    )
    return payload


@router.get("/api/models/download-status")
def model_download_status(api_key: str = Depends(verify_api_key)):
    """Get current model download progress (if any)."""
    agent_status = _get_agent_model_status()
    lifecycle = _model_lifecycle_from_agent_status(agent_status)
    if agent_status and agent_status.get("status") != "idle":
        if _is_cancelled_download_status(agent_status) or _is_stale_terminal_download_status(agent_status):
            idle_status = _idle_download_status(last_terminal_status=agent_status)
            if lifecycle:
                idle_status["modelLifecycle"] = lifecycle
            return idle_status
        return agent_status

    status_path = Path(DATA_DIR) / "model-download-status.json"
    if not status_path.exists():
        bootstrap_status = _read_bootstrap_status_file()
        if _is_cancelled_download_status(bootstrap_status):
            terminal = {**bootstrap_status, "error": "Download cancelled by user; partial file preserved."}
            return _idle_download_status(last_terminal_status=terminal)
        if _is_stale_active_bootstrap_status(bootstrap_status):
            return _stale_bootstrap_download_status(bootstrap_status)
        bootstrap_info = get_bootstrap_status()
        if not bootstrap_info.active:
            status = {"status": "idle", "active": False, "isDownloading": False}
            if lifecycle:
                status["modelLifecycle"] = lifecycle
            return status
        return {
            "status": "downloading",
            "active": True,
            "isDownloading": True,
            "model": bootstrap_info.model_name,
            "percent": bootstrap_info.percent,
            "bytesDownloaded": int((bootstrap_info.downloaded_gb or 0) * 1024**3),
            "bytesTotal": int((bootstrap_info.total_gb or 0) * 1024**3),
            # Same field and unit as host-agent download status; the Models
            # page progress hook reads speedBytesPerSec only.
            "speedBytesPerSec": int((bootstrap_info.speed_mbps or 0) * 1024**2),
            "eta": bootstrap_info.eta_seconds,
        }
    try:
        status = json.loads(status_path.read_text(encoding="utf-8"))
        if _is_cancelled_download_status(status) or _is_stale_terminal_download_status(status):
            idle_status = _idle_download_status(last_terminal_status=status)
            if lifecycle:
                idle_status["modelLifecycle"] = lifecycle
            return idle_status
        if lifecycle:
            status["modelLifecycle"] = lifecycle
        return status
    except (json.JSONDecodeError, OSError):
        status = {"status": "idle"}
        if lifecycle:
            status["modelLifecycle"] = lifecycle
        return status


def _idle_download_status(last_terminal_status: Optional[dict] = None) -> dict:
    status = {"status": "idle", "active": False, "isDownloading": False}
    if last_terminal_status:
        status["lastTerminalStatus"] = last_terminal_status
    return status


def _parse_status_updated_at(value: Any) -> Optional[datetime]:
    if not isinstance(value, str) or not value.strip():
        return None
    try:
        parsed = datetime.fromisoformat(value.strip().replace("Z", "+00:00"))
    except ValueError:
        return None
    if parsed.tzinfo is None:
        parsed = parsed.replace(tzinfo=timezone.utc)
    return parsed.astimezone(timezone.utc)


def _is_stale_terminal_download_status(status: Any) -> bool:
    if not isinstance(status, dict):
        return False
    key = str(status.get("status") or "").casefold()
    if key not in {"failed", "error", "cancelled", "canceled"}:
        return False
    updated_at = _parse_status_updated_at(status.get("updatedAt"))
    if not updated_at:
        return False
    age = (datetime.now(timezone.utc) - updated_at).total_seconds()
    return age > _STALE_TERMINAL_DOWNLOAD_STATUS_SECONDS


def _is_cancelled_download_status(status: Any) -> bool:
    if not isinstance(status, dict):
        return False
    key = str(status.get("status") or "").casefold()
    return key in {"cancelled", "canceled"}


def _read_bootstrap_status_file() -> Optional[dict[str, Any]]:
    status_path = Path(DATA_DIR) / "bootstrap-status.json"
    if not status_path.exists():
        return None
    try:
        status = json.loads(status_path.read_text(encoding="utf-8"))
    except (json.JSONDecodeError, OSError):
        return None
    return status if isinstance(status, dict) else None


def _is_stale_active_bootstrap_status(status: Any) -> bool:
    if not isinstance(status, dict):
        return False
    state = str(status.get("status") or "").casefold()
    if state not in _ACTIVE_BOOTSTRAP_STATUSES:
        return False
    updated_at = _parse_status_updated_at(status.get("updatedAt"))
    if not updated_at:
        return False
    age = (datetime.now(timezone.utc) - updated_at).total_seconds()
    return age > _STALE_ACTIVE_BOOTSTRAP_STATUS_SECONDS


def _stale_bootstrap_download_status(status: dict[str, Any]) -> dict[str, Any]:
    return {
        "status": "failed",
        "active": False,
        "isDownloading": False,
        "bootstrapStale": True,
        "model": status.get("model"),
        "percent": status.get("percent"),
        "bytesDownloaded": status.get("bytesDownloaded", 0),
        "bytesTotal": status.get("bytesTotal", 0),
        "speedBytesPerSec": status.get("speedBytesPerSec", 0),
        "eta": status.get("eta"),
        "updatedAt": status.get("updatedAt"),
        "error": "Bootstrap full-model upgrade appears stalled. Run ods restart to resume it.",
    }


def _bootstrap_retry_pending_error(model_name: Any) -> str:
    """Say why downloads wait on the first full model and how to retry it.

    The retry starts with the next ODS start or restart, and nothing else
    told the owner that (a user hit this on three computers with no way on).
    """
    model = str(model_name or "").strip() or "the full model"
    return (f"ODS's first download of {model} stopped before it finished, and it goes before other "
            "model downloads. Restart ODS to retry it (ods restart). The reason is in "
            "logs/model-upgrade.log in your ODS folder.")


def _bootstrap_upgrade_download_conflict() -> dict[str, Any] | None:
    """Return a lifecycle-busy payload when bootstrap upgrade owns download priority."""
    bootstrap_status = _read_bootstrap_status_file()
    if _is_stale_active_bootstrap_status(bootstrap_status):
        target = bootstrap_status.get("model") if bootstrap_status else None
        return {
            "error": _bootstrap_retry_pending_error(target),
            "code": "model_lifecycle_busy",
            "activeOperation": "bootstrap_upgrade_retry_pending",
            "activeTarget": target,
        }

    bootstrap_info = get_bootstrap_status()
    if bootstrap_info.active:
        model = str(bootstrap_info.model_name or "").strip() or "the full model"
        return {
            "error": (f"ODS is still downloading {model}, its first full model. "
                      "Other model downloads can start when it finishes."),
            "code": "model_lifecycle_busy",
            "activeOperation": "bootstrap_upgrade",
            "activeTarget": bootstrap_info.model_name,
        }

    args_path = Path(DATA_DIR) / "bootstrap-upgrade.args"
    if bootstrap_status is None or not args_path.exists():
        return None

    state = str(bootstrap_status.get("status") or "").casefold()
    model_name = str(bootstrap_status.get("model") or "").strip()
    if state not in {"failed", "error"} or not model_name:
        return None
    if "\x00" in model_name or "/" in model_name or "\\" in model_name or Path(model_name).name != model_name:
        return None

    try:
        final_path = (Path(DATA_DIR) / "models" / model_name).resolve()
        models_root = (Path(DATA_DIR) / "models").resolve()
        if not final_path.is_relative_to(models_root):
            return None
        if final_path.exists() and final_path.stat().st_size > 0:
            return None
    except OSError:
        return None

    return {
        "error": _bootstrap_retry_pending_error(model_name),
        "code": "model_lifecycle_busy",
        "activeOperation": "bootstrap_upgrade_retry_pending",
        "activeTarget": model_name,
    }


def _get_agent_model_status(timeout: int = 5) -> Optional[dict]:
    """Return host-agent-normalized model download status when reachable."""
    global _agent_model_status_cache_at, _agent_model_status_cache_value

    now = time.monotonic()
    if now - _agent_model_status_cache_at < _AGENT_MODEL_STATUS_CACHE_TTL_SECONDS:
        return _agent_model_status_cache_value

    with _agent_model_status_cache_lock:
        now = time.monotonic()
        if now - _agent_model_status_cache_at < _AGENT_MODEL_STATUS_CACHE_TTL_SECONDS:
            return _agent_model_status_cache_value

        try:
            status = request_agent_json("GET", "/v1/model/status", timeout=timeout)
        except AgentClientError:
            status = None

        _agent_model_status_cache_value = status
        _agent_model_status_cache_at = time.monotonic()
        return status


def _invalidate_agent_model_status_cache() -> None:
    """Forget the cached host-agent model status.

    Reads hold the lock across the host-agent request, so an in-flight read
    finishes (and caches) before this reset, never after it. Only the
    timestamp is reset: a lock-free reader then sees either the previous
    consistent entry or an expired one, never a torn value.
    """
    global _agent_model_status_cache_at

    with _agent_model_status_cache_lock:
        _agent_model_status_cache_at = 0.0


# Large GGUF downloads can report the row as downloaded before the host-agent
# releases the model_download lifecycle lock. Keep this finite so unrelated
# conflicts still surface, but cover observed 30s+ multipart teardown lag.
_MODEL_DOWNLOAD_BUSY_ACTIVATION_GRACE_SECONDS = 120.0
# The Pixel access monitor re-proves every ~45 s under the model lifecycle lock
# for a few seconds. A switch that lands in that window waits it out instead of
# returning 409 to a Models page that keeps waiting (Strixy, 2026-10-05).
_PIXEL_LIFECYCLE_OPERATIONS = frozenset({
    "pixel_startup_reproof", "pixel_access_mode", "pixel_open_app",
    "pixel_providers", "pixel_settings",
})
_MODEL_PIXEL_BUSY_ACTIVATION_GRACE_SECONDS = 30.0
# Short holds a download, import or delete waits out instead of refusing: the
# Pixel access re-proof above (about every 45 s; a delete that landed on one
# was refused on Strixy, 2026-10-09), and the integrity check a restarted host
# agent runs when it finds a download status left "verifying" (minutes after an
# install, Mac, 2026-10-09). This retry grace is not the whole import deadline:
# metadata preparation and the transport's other timeout phases are separate.
_DOWNLOAD_SHORT_HOLD_OPERATIONS = _PIXEL_LIFECYCLE_OPERATIONS | {"artifact_verification"}
_DOWNLOAD_SHORT_HOLD_GRACE_SECONDS = 30.0
_LIFECYCLE_BUSY_WORDS = {
    "model_download": "downloading another model",
    "artifact_verification": "checking a downloaded model file",
    "model_activation": "switching models",
    "model_delete": "deleting a model",
    "model_recovery": "restoring the previous model",
    "model_runtime": "changing the model runtime",
    "route_migration": "updating the model route",
    "system_update": "installing an update",
    "opencode_setup": "setting up OpenCode",
    "opencode_start": "starting OpenCode",
    "model_profile_recheck": "checking what the running model can do",
    **{operation: "checking Pixel" for operation in _PIXEL_LIFECYCLE_OPERATIONS},
}


def _agent_http_detail(exc: AgentHTTPError) -> Any:
    detail: Any = exc.detail
    try:
        payload = json.loads(exc.response_text)
        if isinstance(payload, dict):
            detail = payload
    except (json.JSONDecodeError, TypeError):
        pass
    return detail


def _format_gb(value: Any) -> str:
    try:
        return f"{max(int(value), 0) / (1024 ** 3):.1f} GB"
    except (TypeError, ValueError, OverflowError):
        return "an unknown amount"


def _insufficient_disk_detail(agent_detail: Any) -> dict[str, Any]:
    """Explain a host-agent disk refusal in words; no download was started."""
    detail = agent_detail if isinstance(agent_detail, dict) else {}
    required = detail.get("requiredBytes")
    free = detail.get("freeBytes")
    margin = detail.get("marginBytes")
    return {
        "code": "insufficient_disk_space",
        "message": (
            f"Not enough free disk space for this model. It needs {_format_gb(required)}, "
            f"ODS keeps {_format_gb(margin)} free, and {_format_gb(free)} is free now. "
            "Delete models you no longer use or free up space on this drive, then retry."
        ),
        "requiredBytes": required,
        "freeBytes": free,
        "marginBytes": margin,
    }


def _is_download_lifecycle_busy(detail: Any) -> bool:
    return (
        isinstance(detail, dict)
        and detail.get("code") == "model_lifecycle_busy"
        and detail.get("activeOperation") == "model_download"
    )


def _is_pixel_lifecycle_busy(detail: Any) -> bool:
    return (
        isinstance(detail, dict)
        and detail.get("code") == "model_lifecycle_busy"
        and detail.get("activeOperation") in _PIXEL_LIFECYCLE_OPERATIONS
    )


def _is_short_lifecycle_hold(detail: Any) -> bool:
    return (
        isinstance(detail, dict)
        and detail.get("code") == "model_lifecycle_busy"
        and detail.get("activeOperation") in _DOWNLOAD_SHORT_HOLD_OPERATIONS
    )


def _lifecycle_busy_detail(detail: dict[str, Any], blocked: str = "this download cannot start yet") -> dict[str, Any]:
    """A busy refusal in words; nothing was started, so retrying is safe."""
    doing = _LIFECYCLE_BUSY_WORDS.get(str(detail.get("activeOperation") or ""), "finishing another model task")
    return {
        **detail,
        "message": f"ODS is {doing} right now, so {blocked}. Try again in a minute.",
    }


def _call_agent_model(
    path: str,
    body: dict,
    timeout: float = 30,
    *,
    retry_download_busy_seconds: float = 0.0,
    retry_pixel_busy_seconds: float = 0.0,
) -> dict:
    """Call the host agent model endpoint."""
    started = time.monotonic()
    deadline = started + max(float(retry_download_busy_seconds or 0.0), 0.0)
    pixel_deadline = started + max(float(retry_pixel_busy_seconds or 0.0), 0.0)
    try:
        while True:
            try:
                return request_agent_json("POST", path, payload=body, timeout=timeout)
            except AgentHTTPError as exc:
                if exc.status_code != 409:
                    raise
                detail = _agent_http_detail(exc)
                if (
                    (retry_download_busy_seconds > 0
                     and _is_download_lifecycle_busy(detail)
                     and time.monotonic() < deadline)
                    or (retry_pixel_busy_seconds > 0
                        and _is_pixel_lifecycle_busy(detail)
                        and time.monotonic() < pixel_deadline)
                ):
                    time.sleep(0.5)
                    continue
                raise HTTPException(status_code=409, detail=detail) from exc
    except AgentHTTPError as exc:
        if exc.status_code == 409:
            raise HTTPException(status_code=409, detail=_agent_http_detail(exc)) from exc
        if exc.status_code == 400:
            raise HTTPException(status_code=400, detail=_agent_http_detail(exc)) from exc
        detail = _agent_http_detail(exc)
        if (path == "/v1/model/delete" and exc.status_code == 500
                and isinstance(detail, dict)
                and detail.get("code") in {
                    "model_delete_recovery_required", "model_delete_cleanup_pending", "model_delete_status_update_failed",
                }):
            # Keep the existing gateway status while carrying the deletion
            # outcome and recovery instruction to the Library consumer.
            raise HTTPException(status_code=502, detail=detail) from exc
        if exc.status_code == 507:
            raise HTTPException(
                status_code=507,
                detail=_insufficient_disk_detail(_agent_http_detail(exc)),
            ) from exc
        raise HTTPException(status_code=502, detail=exc.detail) from exc
    except AgentUnavailable as exc:
        raise HTTPException(status_code=503, detail=f"Host agent unreachable: {exc}") from exc
    except AgentProtocolError as exc:
        raise HTTPException(status_code=502, detail=f"Invalid host agent response: {exc}") from exc


def _request_agent_download(payload: dict) -> dict:
    """Start a host-agent download; wait out short holds, refuse longer ones in words."""
    return _request_agent_waiting_short_holds("/v1/model/download", payload, "this download cannot start yet")


def _request_agent_waiting_short_holds(path: str, payload: dict, blocked: str,
                                       grace_seconds: float = _DOWNLOAD_SHORT_HOLD_GRACE_SECONDS) -> dict:
    """Call a host-agent model route; wait out short holds, refuse longer ones in words."""
    remaining = grace_seconds
    deadline = time.monotonic() + remaining
    while True:
        try:
            return _call_agent_model(path, payload, timeout=remaining)
        except HTTPException as exc:
            detail = exc.detail
            if exc.status_code != 409 or not isinstance(detail, dict) \
                    or detail.get("code") != "model_lifecycle_busy":
                raise
            remaining = deadline - time.monotonic()
            if _is_short_lifecycle_hold(detail) and remaining > 0:
                time.sleep(min(0.5, remaining))
                remaining = deadline - time.monotonic()
                if remaining > 0:
                    continue
            raise HTTPException(status_code=409, detail=_lifecycle_busy_detail(detail, blocked)) from exc


def _find_model_in_library(model_id: str) -> Optional[dict]:
    """Look up a model by ID in the library catalog."""
    for model in _load_library():
        if model.get("id") == model_id:
            return model
    return None


def _local_gguf_filename_from_id(model_id: str) -> str | None:
    """Map a dashboard fallback model ID to a local GGUF filename."""
    token = str(model_id or "").strip()
    # A retired Lemonade id still names its GGUF for one release.
    for prefix in ("extra.", "user."):
        if token.lower().startswith(prefix):
            token = token[len(prefix):]
            break
    if not token or any(sep in token for sep in ("/", "\\", "\x00")):
        return None
    filename = token if token.lower().endswith(".gguf") else f"{token}.gguf"
    if filename.lower().endswith(".part") or Path(filename).name != filename:
        return None
    return filename


def _resolve_local_gguf_filename(model_id: str) -> str | None:
    candidate = _local_gguf_filename_from_id(model_id)
    if not candidate:
        return None

    candidate_lower = candidate.lower()
    candidate_stem = Path(candidate).stem.lower()
    exact_matches: list[Path] = []
    stem_matches: list[Path] = []
    logical_matches: list[Path] = []
    candidate_logical = _local_model_name_from_gguf(candidate).lower()
    try:
        for path in _installed_model_paths().values():
            if not _is_final_gguf_file(path):
                continue
            if path.name.lower() == candidate_lower:
                exact_matches.append(path)
            elif path.stem.lower() == candidate_stem:
                stem_matches.append(path)
            elif _local_model_name_from_gguf(path.name).lower() == candidate_logical:
                logical_matches.append(path)
    except OSError as exc:
        logger.warning("Failed to resolve local GGUF %s: %s", model_id, exc)
        return None

    matches = exact_matches or stem_matches or logical_matches
    if len(matches) == 1:
        return matches[0].name
    if len(matches) > 1:
        logger.warning("Ambiguous local GGUF model id %s matched %s", model_id, [p.name for p in matches])
    return None


def _find_local_gguf_model(model_id: str) -> Optional[dict]:
    """Return a synthetic activation record for a manually installed GGUF."""
    gguf_file = _resolve_local_gguf_filename(model_id)
    if not gguf_file:
        return None
    target = _installed_model_path(gguf_file)
    if target is None or not _is_final_gguf_file(target):
        return None

    context_length = 32768
    context_found = False
    for reader in (read_env_file_value, read_env_value):
        for key in ("CTX_SIZE", "MAX_CONTEXT"):
            try:
                value = int(reader(key, INSTALL_DIR) or 0)
            except (TypeError, ValueError):
                continue
            if value > 0:
                context_length = value
                context_found = True
                break
        if context_found:
            break

    model_name = _local_model_name_from_gguf(gguf_file)
    return {
        "id": model_name,
        "gguf_file": gguf_file,
        "llm_model_name": model_name,
        "context_length": context_length,
        "runtime_profiles": [],
        "local": True,
    }


def _find_loadable_model(model_id: str) -> Optional[dict]:
    return _find_model_in_library(model_id) or _find_local_gguf_model(model_id)


def _find_normalized_model(model_id: str) -> Optional[dict]:
    return find_catalog_model(load_model_catalog(INSTALL_DIR), model_id, None)


def loaded_model_display_name(loaded_model: str | None) -> str | None:
    """The name the Models page shows for the model a runtime serves.

    A runtime serves the model file name as its id (llama-server --alias), so
    a status reader would show a file name. The first catalog entry matching
    it is the one the Models page shows as loaded; None when none matches.
    """
    if not loaded_model:
        return None
    library = _load_library()
    if LLM_BACKEND == "vllm":
        # vLLM serves --served-model-name; its entry has no GGUF to match.
        entry = next((raw for raw in library if is_vllm_entry(raw)
                      and raw.get("served_model_name") == loaded_model), None)
        return entry.get("name") if entry else None
    catalog = [entry for entry in map(normalize_catalog_entry, library) if entry is not None]
    entry = find_catalog_model(catalog, loaded_model, loaded_model)
    return entry["name"] if entry else None


def _external_api_host(url: str) -> str | None:
    """The host (and port) of the API URL, without anything else."""
    try:
        parsed = urlsplit(url.strip())
        host = parsed.hostname
        port = parsed.port
    except ValueError:
        return None
    if not host or parsed.scheme not in {"http", "https"}:
        return None
    return f"{host}:{port}" if port else host


async def _fetch_llama_loaded_model(host: str, port: int) -> str | None:
    base_url = _configured_llm_base_url(host, port)
    # A generic OpenAI-compatible server lists every model it can serve with no
    # loaded status: its first entry is not the active model.
    external_compatible = (
        LLM_BACKEND == "external"
        and os.environ.get("EXTERNAL_LLM_PROVIDER", "").strip().lower() == "openai-compatible"
    )
    async with httpx.AsyncClient(timeout=10.0) as client:
        try:
            resp = await client.get(f"{base_url}/v1/models")
            resp.raise_for_status()
            data = resp.json().get("data") or []
            for model in data:
                status = model.get("status", {})
                if isinstance(status, dict) and status.get("value") == "loaded":
                    return model.get("id")
            if data and data[0].get("id") and not external_compatible:
                return data[0]["id"]
        except (httpx.HTTPError, ValueError):
            pass
        if external_compatible:
            return None

        try:
            resp = await client.get(f"{base_url}/props")
            resp.raise_for_status()
            props = resp.json()
            if props.get("model_alias"):
                return props["model_alias"]
            if props.get("model_path"):
                return Path(props["model_path"]).name
        except (httpx.HTTPError, ValueError):
            return None
    return None


def _benchmark_measurement(data: Any, wall_seconds: float) -> tuple[int, float, str]:
    """Use a matched pair of request measurements; server totals include peers."""
    data = data if isinstance(data, dict) else {}
    timings = data.get("timings")
    if isinstance(timings, dict):
        tokens = timings.get("predicted_n")
        milliseconds = timings.get("predicted_ms")
        if (type(tokens) is int and tokens > 0
                and type(milliseconds) in (int, float)
                and math.isfinite(milliseconds) and milliseconds > 0):
            return tokens, milliseconds / 1000.0, "request timings"
    usage = data.get("usage")
    tokens = usage.get("completion_tokens") if isinstance(usage, dict) else None
    if type(tokens) is int and tokens > 0:
        return tokens, wall_seconds, "request usage / wall time"
    raise HTTPException(
        status_code=502,
        detail="Benchmark completed but no valid request token count was reported; result was not saved",
    )


async def _run_current_model_benchmark(model_id: str, max_tokens: int) -> dict:
    service = SERVICES.get("llama-server")
    if not service:
        raise HTTPException(status_code=503, detail="llama-server service is not configured")
    host = service.get("host", "llama-server")
    port = int(service.get("port", 8080))

    loaded_model = await get_loaded_model()
    if not loaded_model:
        loaded_model = await _fetch_llama_loaded_model(host, port)
    if not loaded_model:
        loaded_model = _read_active_model() or read_env_value("LLM_MODEL", INSTALL_DIR)
    if not loaded_model:
        raise HTTPException(status_code=503, detail="llama-server is not reporting a loaded model")

    gpu_info = await asyncio.to_thread(get_gpu_info)
    context_size = await get_llama_context_size(model_hint=loaded_model)
    metrics = await _await_or_default(
        get_llama_metrics(model_hint=loaded_model),
        {"tokens_per_second": 0},
        "llama metrics",
    )
    payload = await asyncio.to_thread(
        build_models_payload,
        gpu_info,
        loaded_model,
        _newly_measured_tps(metrics, loaded_model),
        INSTALL_DIR,
        DATA_DIR,
        context_size,
        catalog=_load_library(),
        downloaded_files_override=_installed_model_paths(),
    )
    target = next((m for m in payload["models"] if m["id"] == model_id), None)
    if target is None:
        raise HTTPException(status_code=404, detail="Unknown model")
    if target["status"] != "loaded":
        raise HTTPException(status_code=409, detail="Load the model before benchmarking it")

    max_tokens = max(32, min(int(max_tokens or 128), 512))
    prompt = (
        "You are benchmarking local inference. Write a concise technical explanation "
        "of why local LLM throughput depends on model size, quantization, backend, "
        "context length, and GPU memory bandwidth. Continue until the token budget ends."
    )

    # A Windows-hosted llama-server requires its key, which only LiteLLM
    # holds; every other runtime is measured directly.
    url, headers = f"http://{host}:{port}/v1/chat/completions", None
    if read_env_value("AMD_INFERENCE_LOCATION", INSTALL_DIR).strip().lower() == "host":
        gateway_key = read_env_value("LITELLM_KEY", INSTALL_DIR) or read_env_value("LITELLM_MASTER_KEY", INSTALL_DIR)
        url = "http://litellm:4000/v1/chat/completions"
        headers = {"Authorization": f"Bearer {gateway_key}"} if gateway_key else None
    started = time.perf_counter()
    async with httpx.AsyncClient(timeout=max(60.0, max_tokens * 3.0)) as client:
        resp = await client.post(
            url,
            json={
                "model": loaded_model,
                "messages": [{"role": "user", "content": prompt}],
                "temperature": 0,
                "max_tokens": max_tokens,
                "stream": False,
            },
            headers=headers,
        )
        resp.raise_for_status()
        response_data = resp.json()
    wall_seconds = max(time.perf_counter() - started, 0.001)
    generated, generate_seconds, method = _benchmark_measurement(response_data, wall_seconds)

    tokens_per_second = round(generated / generate_seconds, 2)
    if not is_plausible_single_request_tps(tokens_per_second):
        raise HTTPException(
            status_code=502,
            detail="Benchmark returned implausible single-request throughput; result was not saved",
        )
    if gpu_info:
        gguf_path = _installed_model_path(target["gguf"]) if target.get("gguf") else None
        signature = build_sample_signature(target, gpu_info, context_size, INSTALL_DIR, gguf_path)
        for sample_name in {model_id, loaded_model, target.get("gguf") or "", target.get("llmModelName") or ""}:
            if not sample_name:
                continue
            await asyncio.to_thread(
                record_model_performance,
                sample_name,
                gpu_info.name,
                gpu_info.gpu_backend,
                tokens_per_second,
                model_id=signature.get("model_id"),
                gguf=signature.get("gguf"),
                quantization=signature.get("quantization"),
                architecture=signature.get("architecture"),
                context_length=signature.get("context_length"),
                decode_read_mb=signature.get("decode_read_mb"),
                vram_total_mb=signature.get("vram_total_mb"),
                os_name=signature.get("os"),
                flags=signature.get("flags"),
                source="local_benchmark",
            )

    return {
        "model": model_id,
        "loadedModel": loaded_model,
        "contextLength": context_size or target.get("contextLength"),
        "tokensPerSecond": tokens_per_second,
        "generatedTokens": int(generated),
        "generateSeconds": round(generate_seconds, 3),
        "wallSeconds": round(wall_seconds, 3),
        "source": "local_benchmark",
        "method": method,
    }


@router.post("/api/models/{model_id}/download")
def download_model(model_id: str, api_key: str = Depends(verify_api_key)):
    """Start downloading a model from HuggingFace."""
    model = _find_model_in_library(model_id)
    if model is None:
        raise HTTPException(status_code=404, detail=f"Model '{model_id}' not found in library")
    _refuse_vllm_entry(model, "The ODS installer downloads it when it sets that engine up.")

    bootstrap_conflict = _bootstrap_upgrade_download_conflict()
    if bootstrap_conflict is not None:
        raise HTTPException(
            status_code=409,
            detail={**bootstrap_conflict, "requestedModelId": model_id},
        )

    # Split-file models provide gguf_parts; imports may carry a vision projector.
    return _request_agent_download(_hf_download_payload(model))


@router.post("/api/models/download/cancel")
def cancel_download(api_key: str = Depends(verify_api_key)):
    """Cancel an in-progress model download."""
    bootstrap = _read_bootstrap_status_file()
    if bootstrap and bootstrap.get("status") in _ACTIVE_BOOTSTRAP_STATUSES:
        if bootstrap.get("status") not in {"starting", "downloading"}:
            raise HTTPException(status_code=409, detail="The full model has finished downloading and is being activated. It cannot be cancelled now.")
        generation = bootstrap.get("downloadGeneration")
        if not valid_generation(generation):
            raise HTTPException(status_code=409, detail="This full-model download worker does not support cancellation acknowledgements. It is still running.")
        # This is a nonce, not a secret. The native owner must be able to read
        # a request written by the API container; only the API writes requests.
        directory = Path(DATA_DIR)
        request_path = directory / "bootstrap-download.cancel.json"
        temporary = None
        try:
            fd, temporary = tempfile.mkstemp(prefix=".bootstrap-cancel-", dir=directory)
            with os.fdopen(fd, "w", encoding="utf-8") as handle:
                json.dump({"generation": generation}, handle)
                handle.flush()
                os.fchmod(handle.fileno(), 0o644)
                os.fsync(handle.fileno())
            os.replace(temporary, request_path)
        except OSError:
            raise HTTPException(status_code=503, detail="The download cancellation request could not be delivered. The download may still be running.") from None
        finally:
            if temporary is not None:
                try:
                    os.unlink(temporary)
                except FileNotFoundError:
                    pass
        deadline = time.monotonic() + 10
        while time.monotonic() < deadline:
            current = _read_bootstrap_status_file()
            if not current or current.get("downloadGeneration") != generation:
                raise HTTPException(status_code=409, detail="The download changed before cancellation was acknowledged. Check its current progress.")
            state = current.get("status")
            if state == "cancelled":
                return {"status": "cancelled", "cancelled": True, "downloadGeneration": generation}
            if state not in {"starting", "downloading"}:
                raise HTTPException(status_code=409, detail="The download finished or changed before it could be cancelled. Check its current progress.")
            time.sleep(0.2)
        raise HTTPException(status_code=409, detail="The download has not acknowledged cancellation yet. Check progress before retrying.")
    result = _call_agent_model("/v1/model/download/cancel", {})
    return result


def _model_recovery_projection(value):
    phases = {'idle', 'completed', 'prepared', 'held', 'applying', 'applied', 'committing', 'rolling-back', 'unavailable'}
    if (type(value) is not dict or type(value.get('pending')) is not bool
            or value.get('phase') not in phases
            or value['pending'] != (value['phase'] not in ('idle', 'completed'))):
        raise ValueError('invalid-model-recovery')
    transaction = value.get('transactionId')
    if (transaction is not None and (type(transaction) is not str or re.fullmatch('[a-f0-9]{64}', transaction) is None)
            or value['phase'] not in ('idle', 'unavailable') and transaction is None):
        raise ValueError('invalid-model-recovery')
    result = {'pending': value['pending'], 'phase': value['phase'], 'transactionId': transaction}
    if value.get('outcome') in ('commit', 'rollback') and value['phase'] == 'completed':
        result['outcome'] = value['outcome']
    if value.get('reason') in ('model-recovery-proof-required', 'model-recovery-unavailable'):
        result['reason'] = value['reason']
    # The agent offers the owner a release without the live proof only for a
    # switch that changed nothing (fleet row 27).
    if value['pending'] and type(value.get('releasable')) is bool:
        result['releasable'] = value['releasable']
    return result


_RECOVERY_REQUESTS: tuple[dict[str, bool], ...] = ({}, {'releaseUnverified': True})


def _model_recovery_request(method, body=None):
    try:
        value = request_agent_json(method, '/v1/model/recovery' if method == 'GET' else '/v1/model/recover',
                                   payload=None if method == 'GET' else body, timeout=5 if method == 'GET' else 400)
        return _model_recovery_projection(value)
    except AgentHTTPError as exc:
        if exc.status_code in (409, 503):
            try:
                return JSONResponse(_model_recovery_projection(_agent_http_detail(exc)), status_code=exc.status_code,
                                    headers={'Cache-Control': 'no-store'})
            except ValueError:
                pass
        raise HTTPException(status_code=503, detail='Model recovery is unavailable. No new model switch was started.') from None
    except (AgentClientError, ValueError):
        raise HTTPException(status_code=503, detail='Model recovery could not be confirmed. Refresh before retrying.') from None


@router.get('/api/models/recovery')
def model_recovery_status(api_key: str = Depends(verify_api_key)):
    value = _model_recovery_request('GET')
    return value if isinstance(value, JSONResponse) else JSONResponse(value, headers={'Cache-Control': 'no-store'})


@router.post('/api/models/recovery')
def recover_model_switch(body: dict | None = Body(default=None), api_key: str = Depends(verify_api_key)):
    if body is None or not any(body == allowed and all(type(body[key]) is type(value) for key, value in allowed.items())
                               for allowed in _RECOVERY_REQUESTS):
        raise HTTPException(status_code=400, detail='Recovery accepts {} or {"releaseUnverified": true} only.')
    try:
        value = _model_recovery_request('POST', dict(body))
    finally:
        # Recovery can restore the earlier runtime.
        _forget_model_management()
    return value if isinstance(value, JSONResponse) else JSONResponse(value, headers={'Cache-Control': 'no-store'})


_RETIRED_ADOPTION = {
    "error": "Adopting a model loaded in Lemonade was removed",
    "code": "external_lemonade_removed",
    "hint": ("ODS runs llama-server for every managed runtime. To use your own Lemonade, "
             "change its model there and select it with the installer's --external-llm-* options."),
}


@router.get('/api/models/external-observation')
def external_model_observation(api_key: str = Depends(verify_api_key)):
    """Removed with Lemonade (round F); answers 410 for one release."""
    raise HTTPException(status_code=410, detail=_RETIRED_ADOPTION)


@router.post('/api/models/external-adopt')
def adopt_external_model(api_key: str = Depends(verify_api_key)):
    """Removed with Lemonade (round F); answers 410 for one release."""
    raise HTTPException(status_code=410, detail=_RETIRED_ADOPTION)


@router.post("/api/models/runtime/{operation}")
def manage_model_runtime(operation: str, body: dict | None = Body(default=None),
                         api_key: str = Depends(verify_api_key)):
    if operation not in {"stop", "start"} or body not in (None, {}):
        raise HTTPException(status_code=400, detail="A start or stop operation with an empty body is required")
    if pixel_stream_active():
        raise HTTPException(status_code=409, detail="Stop the active Portal response before changing its runtime")
    try:
        value = _call_agent_model(f"/v1/model/runtime/{operation}", {}, timeout=1200)
    finally:
        _invalidate_agent_model_status_cache()
        _forget_model_management()
    expected = "started" if operation == "start" else "stopped"
    if not isinstance(value, dict) or value.get("status") != expected:
        raise HTTPException(status_code=502, detail="The runtime operation was not confirmed; refresh its status")
    return JSONResponse({"status": expected}, headers={"Cache-Control": "no-store"})


@router.post("/api/models/{model_id}/load")
def load_model(
    model_id: str,
    body: dict[str, Any] | None = Body(default=None),
    api_key: str = Depends(verify_api_key),
):
    """Activate a model — update config and restart llama-server."""
    return _activate_model(model_id, body)


def _activate_model(model_id: str, body: dict[str, Any] | None, *, chat_template_override: str | None = None):
    """Run ``model_id`` through the host agent's activation transaction.

    ``chat_template_override`` (any-model WP5) asks for the model's fixed
    chat template; such a request always restarts the runtime, even when the
    model already runs.
    """
    mode_denial = _model_activation_mode_denial(
        ODS_MODE_EFFECTIVE,
        _configured_ods_mode(),
        LLM_BACKEND,
    )
    if mode_denial is not None:
        raise HTTPException(
            status_code=409,
            detail={**mode_denial, "requestedModelId": model_id},
        )
    if _windows_hosted_runtime() and not _model_management().get("canActivate"):
        raise HTTPException(
            status_code=409,
            detail={
                "error": "This installation cannot change the model runtime on the Windows host right now",
                "code": "external_runtime_unmanaged",
                "requestedModelId": model_id,
            },
        )

    model = _find_loadable_model(model_id)
    if model is None:
        raise HTTPException(status_code=404, detail=f"Model '{model_id}' not found in library or local GGUF files")
    _refuse_vllm_entry(model, "Only the ODS installer sets that engine up and runs it.")

    requested_context = _requested_activation_context(body)
    already_active, loaded_model = _already_active_model(model_id, model)
    served_context = _verified_activation_context(loaded_model) if already_active else None
    # Without an explicit context, a switch serves what the installer would
    # serve on this hardware (see _policy_activation_context), starting from
    # the context this model already runs at, or the installer's pick.
    policy_context = None
    if requested_context is None:
        if _configured_model_identity_matches(model):
            preferred = _configured_context_length()
        else:
            preferred = _recommended_model_context(model)
        policy_context = _policy_activation_context(model_id, preferred)
    # An idempotent reload keeps a running model as it is, unless it runs
    # below the Hermes floor and the floor fits: that model cannot serve
    # ODS Talk, so the reload repairs it.
    raise_below_floor = (
        requested_context is None
        and policy_context is not None
        and served_context is not None
        and served_context < HERMES_MIN_CONTEXT <= policy_context
    )
    if chat_template_override is None and already_active and not raise_below_floor and (
        requested_context is None
        or requested_context == served_context
    ):
        response: dict[str, Any] = {
            "status": "already_active",
            "model_id": model_id,
            "loadedModel": loaded_model,
        }
        configured_context = _configured_context_length()
        if configured_context is not None:
            response["context_length"] = configured_context
        return response

    if pixel_stream_active():
        raise HTTPException(
            status_code=409,
            detail={
                "code": "pixel_chat_active",
                "message": "Portal is working. Stop the active response before changing models.",
                "requestedModelId": model_id,
            },
        )

    bootstrap_conflict = _bootstrap_upgrade_download_conflict()
    if bootstrap_conflict is not None:
        raise HTTPException(
            status_code=409,
            detail={**bootstrap_conflict, "requestedModelId": model_id},
        )

    # Activation includes downstream synchronization and a bounded rollback.
    activation_body: dict[str, Any] = {"model_id": model_id}
    activation_context = requested_context
    if activation_context is None and policy_context is not None:
        activation_context = policy_context
    if (
        activation_context is None
        and (
            _configured_model_identity_matches(model)
            or (
                loaded_model
                and (_model_name_tokens(loaded_model) & _catalog_model_tokens(model))
            )
        )
    ):
        # A matching live backend may still require reconciliation when its
        # activation receipt is absent or stale (for example immediately after
        # bootstrap promotion).  Preserve the verified runtime context across
        # that repair.  Otherwise the host agent falls back to the catalog's
        # conservative default and can silently shrink a 64K Hermes-capable
        # runtime to 32K during an idempotent dashboard reload.
        configured_context = _configured_context_length()
        if (
            configured_context is not None
            and _MIN_MODEL_CONTEXT <= configured_context <= _MAX_MODEL_CONTEXT
        ):
            activation_context = configured_context
    if activation_context is not None:
        activation_body["context_length"] = activation_context
    if chat_template_override is not None:
        activation_body["chat_template_override"] = chat_template_override
    try:
        result = _call_agent_model(
            "/v1/model/activate",
            activation_body,
            timeout=2700,
            retry_download_busy_seconds=_MODEL_DOWNLOAD_BUSY_ACTIVATION_GRACE_SECONDS,
            retry_pixel_busy_seconds=_MODEL_PIXEL_BUSY_ACTIVATION_GRACE_SECONDS,
        )
    finally:
        # A status read cached while the activation ran still reports its
        # lifecycle as active. Drop it so the dashboard's confirming poll,
        # sent as soon as this response lands, sees the settled state.
        _invalidate_agent_model_status_cache()
        _forget_model_management()
    return result


@router.get("/api/models/{model_id}/profile")
async def model_profile(model_id: str, api_key: str = Depends(verify_api_key)):
    """What a model was measured to do on this machine (PLAN WP3); advisory."""
    try:
        return await asyncio.to_thread(
            request_agent_json, "GET", "/v1/model/profile", params={"model": model_id}, timeout=10,
        )
    except AgentHTTPError as exc:
        raise HTTPException(status_code=502, detail=_agent_http_detail(exc)) from exc
    except AgentClientError as exc:
        raise HTTPException(status_code=503, detail=f"Host agent unreachable: {exc}") from exc


# The probe battery's own budget (120 s) plus the runtime's /props read.
_MODEL_PROFILE_RECHECK_TIMEOUT_SECONDS = 180


@router.post("/api/models/{model_id}/profile/recheck")
def recheck_model_profile(model_id: str, api_key: str = Depends(verify_api_key)):
    """Measure the running model again, ignoring its stored profile."""
    return _call_agent_model(
        "/v1/model/profile/recheck", {"model": model_id}, timeout=_MODEL_PROFILE_RECHECK_TIMEOUT_SECONDS,
    )


# A fixed chat template's id, as config/chat-templates/index.json spells it.
_CHAT_TEMPLATE_ID_RE = re.compile(r"[a-z0-9](?:[a-z0-9.-]{0,62}[a-z0-9])?")


def _chat_template_unavailable_reason() -> str | None:
    """Why this runtime cannot load a fixed chat template, or None when it can."""
    mode = read_live_env_values(("AMD_INFERENCE_RUNTIME_MODE",)).get("AMD_INFERENCE_RUNTIME_MODE")
    if str(mode or "").strip().casefold() in _LEGACY_WINDOWS_NATIVE_MODES or _windows_hosted_runtime():
        return "The Windows model runtime on this machine cannot use a fixed chat template yet. Nothing was changed."
    return None


@router.post("/api/models/{model_id}/chat-template")
def run_with_fixed_chat_template(
    model_id: str,
    body: dict[str, Any] | None = Body(default=None),
    api_key: str = Depends(verify_api_key),
):
    """Run a model with ODS's fixed chat template for it (any-model WP5).

    The host agent accepts the template only when its index fixes exactly the
    template this model was measured with. A running model keeps its context.
    """
    override = body.get("override") if isinstance(body, dict) and set(body) == {"override"} else None
    if not isinstance(override, str) or not _CHAT_TEMPLATE_ID_RE.fullmatch(override):
        raise HTTPException(status_code=400, detail='Ask for a fixed chat template as {"override": "<id>"}.')
    reason = _chat_template_unavailable_reason()
    if reason is not None:
        raise HTTPException(status_code=409, detail={
            "code": "chat_template_override_unsupported", "message": reason, "requestedModelId": model_id,
        })
    model = _find_loadable_model(model_id)
    activation = None
    if model is not None and _configured_model_identity_matches(model):
        configured = _configured_context_length()
        if configured is not None and _MIN_MODEL_CONTEXT <= configured <= _MAX_MODEL_CONTEXT:
            activation = {"context_length": configured}
    return _activate_model(model_id, activation, chat_template_override=override)


@router.post("/api/models/{model_id}/benchmark")
async def benchmark_model(model_id: str, body: dict[str, Any] | None = None, api_key: str = Depends(verify_api_key)):
    """Benchmark only the currently loaded model on this machine."""
    max_tokens = 128
    if isinstance(body, dict) and body.get("max_tokens"):
        try:
            max_tokens = int(body["max_tokens"])
        except (TypeError, ValueError):
            raise HTTPException(status_code=400, detail="max_tokens must be an integer")
    try:
        return await _run_current_model_benchmark(model_id, max_tokens)
    except httpx.HTTPStatusError as exc:
        raise HTTPException(
            status_code=502,
            detail=f"llama-server benchmark request failed: HTTP {exc.response.status_code}",
        ) from exc
    except httpx.HTTPError as exc:
        raise HTTPException(status_code=503, detail=f"llama-server is not reachable for benchmark: {exc}") from exc


@router.delete("/api/models/{model_id}")
def delete_model(model_id: str, api_key: str = Depends(verify_api_key)):
    """Delete a downloaded model file."""
    model = _find_model_in_library(model_id) or _find_local_gguf_model(model_id)
    if model is None:
        raise HTTPException(status_code=404, detail=f"Model '{model_id}' not found in library or local GGUF files")
    _refuse_vllm_entry(model, "The ODS installer manages its files.")

    payload = {
        "gguf_file": model["gguf_file"],
    }
    if model.get("gguf_parts"):
        payload["gguf_parts"] = model["gguf_parts"]
    # 20 s leaves the delete itself inside the Models page's 35 s deadline.
    return _request_agent_waiting_short_holds("/v1/model/delete", payload, "this model cannot be deleted yet",
                                              grace_seconds=20.0)