ODS V3 Pre-Release: Public testing and refinement ahead of the official V3 launch. Turn your PC, Mac, or Linux box into a private AI server.
Python ai-agentsamdcomfyuidockerllama-cppllmlocal-ain8nnvidiaopen-webuiragself-hostedspeech-to-textstrix-halotext-to-speechworkflow-automation
"""Shared helper functions for service health checking, metrics, and system info."""
import asyncio
import hashlib
import json
import logging
import math
import os
import platform
import re
import shutil
import socket
import threading
import time
from pathlib import Path
from typing import Optional
import aiohttp
import httpx
from config import (
SERVICES, INSTALL_DIR, DATA_DIR, LLM_BACKEND, EXTENSIONS_DIR, GPU_BACKEND,
LIBRARY_MANAGEABLE_BUILTINS, get_env_file_read_error, load_extension_manifests, read_live_env_value,
)
from env_values import parse_env_value
from host_metrics import apple_host_metrics, linux_scope, windows_host_metrics
from host_agent_client import AgentClientError, AgentHTTPError, async_request_json as request_agent_json
from host_agent_client import request_json as request_agent_json_sync
from models import ServiceStatus, DiskUsage, ModelInfo, BootstrapStatus
from service_health_dns import ServiceHealthResolver
class _DirSizeCache:
"""Per-path TTL cache for dir_size_gb to avoid repeated rglob walks."""
def __init__(self, ttl: float = 60.0):
self._ttl = ttl
self._store: dict[str, tuple[float, float]] = {}
def get(self, path: Path) -> float | None:
key = str(path.resolve())
entry = self._store.get(key)
if entry is None:
return None
expires_at, value = entry
if time.monotonic() > expires_at:
del self._store[key]
return None
return value
def set(self, path: Path, value: float):
now = time.monotonic()
expired_keys = [k for k, (expires_at, _) in self._store.items() if now > expires_at]
for k in expired_keys:
del self._store[k]
key = str(path.resolve())
if len(self._store) >= 1000 and key not in self._store:
oldest_key = next(iter(self._store))
del self._store[oldest_key]
self._store[key] = (now + self._ttl, value)
def invalidate(self, path: Path) -> None:
self._store.pop(str(path.resolve()), None)
def clear(self) -> None:
self._store.clear()
_dir_size_cache = _DirSizeCache()
# Every managed runtime is upstream llama-server (OpenAI routes under /v1).
_LLM_API_PREFIX = "/v1"
logger = logging.getLogger(__name__)
# --- Shared HTTP sessions (connection pooling) ---
# Re-using sessions avoids creating/destroying TCP connections every
# poll cycle and prevents file-descriptor exhaustion.
_aio_session: Optional[aiohttp.ClientSession] = None
_aio_session_lock: Optional[asyncio.Lock] = None
_health_resolver: Optional[ServiceHealthResolver] = None
_HEALTH_TIMEOUT = aiohttp.ClientTimeout(total=30)
# Short timeout for the catalog fan-out: one slow probe must not stall the
# whole Extensions page (frontend aborts after 8 s).
_CATALOG_HEALTH_TIMEOUT = aiohttp.ClientTimeout(total=5)
def _get_aio_session_lock() -> asyncio.Lock:
global _aio_session_lock
if _aio_session_lock is None:
_aio_session_lock = asyncio.Lock()
return _aio_session_lock
async def _get_aio_session() -> aiohttp.ClientSession:
"""Return (and lazily create) a module-level aiohttp session."""
global _aio_session, _health_resolver
if _aio_session is not None and not _aio_session.closed:
return _aio_session
async with _get_aio_session_lock():
if _aio_session is None or _aio_session.closed:
if _health_resolver is not None:
await _health_resolver.close()
_health_resolver = ServiceHealthResolver()
_aio_session = aiohttp.ClientSession(
timeout=_HEALTH_TIMEOUT,
connector=aiohttp.TCPConnector(family=socket.AF_INET, resolver=_health_resolver),
)
return _aio_session
async def shutdown_service_health_client() -> None:
"""Close health sockets and cancel queued resolver work at app shutdown."""
global _aio_session, _health_resolver, _aio_session_lock
if _aio_session is not None:
await _aio_session.close()
_aio_session = None
if _health_resolver is not None:
await _health_resolver.close()
_health_resolver = None
_aio_session_lock = None
# Shared httpx client for llama-server requests (connection pooling)
_httpx_client: Optional[httpx.AsyncClient] = None
_httpx_client_lock: Optional[asyncio.Lock] = None
def _get_httpx_client_lock() -> asyncio.Lock:
global _httpx_client_lock
if _httpx_client_lock is None:
_httpx_client_lock = asyncio.Lock()
return _httpx_client_lock
async def _get_httpx_client() -> httpx.AsyncClient:
"""Return (and lazily create) a module-level httpx async client."""
global _httpx_client
if _httpx_client is not None and not _httpx_client.is_closed:
return _httpx_client
async with _get_httpx_client_lock():
if _httpx_client is None or _httpx_client.is_closed:
_httpx_client = httpx.AsyncClient(timeout=5.0)
return _httpx_client
async def shutdown_llm_client() -> None:
"""Close the pooled LLM client after application users have stopped."""
global _httpx_client, _httpx_client_lock
if _httpx_client is not None:
await _httpx_client.aclose()
_httpx_client = None
_httpx_client_lock = None
def _service_status_from_config(service_id: str, config: dict, status: str) -> ServiceStatus:
return ServiceStatus(
id=service_id, name=config["name"], port=config["port"],
external_port=config.get("external_port", config["port"]),
status=status, response_time_ms=None,
)
async def _check_tailscale_health(service_id: str, config: dict) -> ServiceStatus:
"""Map the host-agent Tailscale snapshot into a service health status.
Tailscale has no HTTP port to poll. Treat an absent container as
not_deployed so the optional Remote Access feature does not make a fresh
local install look degraded.
"""
try:
payload = await request_agent_json("GET", "/v1/tailscale/status", timeout=5)
except AgentClientError:
return _service_status_from_config(service_id, config, "not_deployed")
if not payload.get("running"):
return _service_status_from_config(service_id, config, "not_deployed")
if payload.get("authenticated"):
return _service_status_from_config(service_id, config, "healthy")
return _service_status_from_config(service_id, config, "unhealthy")
# Last OpenCode lifecycle reported by the host agent. The dashboard uses it to
# tell "never set up" from "installed but stopped"; a bare port probe cannot.
_opencode_lifecycle: Optional[dict] = None
_OPENCODE_STATE_STATUS = {
"running": "healthy",
"starting": "degraded",
"installing": "degraded",
"stopped": "down",
"not_installed": "not_deployed",
}
def get_opencode_lifecycle() -> Optional[dict]:
"""Return the most recent OpenCode lifecycle snapshot, if any."""
return _opencode_lifecycle
async def _check_opencode_health(service_id: str, config: dict) -> ServiceStatus:
"""Map the host agent's OpenCode lifecycle onto the service vocabulary.
``running`` -> healthy, ``starting``/``installing`` -> degraded,
``stopped`` -> down (installed but not running), ``not_installed`` ->
not_deployed. Older host agents without the lifecycle route fall back to
the loopback port proof.
"""
global _opencode_lifecycle
try:
payload = await request_agent_json("GET", "/v1/opencode/status", timeout=10)
except AgentHTTPError as exc:
_opencode_lifecycle = None
if exc.status_code == 404:
return await _check_host_port_health(service_id, config)
return _service_status_from_config(service_id, config, "down")
except AgentClientError:
_opencode_lifecycle = None
return _service_status_from_config(service_id, config, "down")
status = _OPENCODE_STATE_STATUS.get(payload.get("state"))
if status is None:
_opencode_lifecycle = None
return _service_status_from_config(service_id, config, "down")
_opencode_lifecycle = payload
response_time = payload.get("responseTimeMs")
return ServiceStatus(
id=service_id,
name=config["name"],
port=config["port"],
external_port=config.get("external_port", config["port"]),
status=status,
response_time_ms=response_time if isinstance(response_time, (int, float)) else None,
)
async def _check_pixel_ingress_health(service_id: str, config: dict) -> ServiceStatus:
"""Prove the socket-only Pixel host ingress through Pixel Edge.
The ingress listens only on a Unix socket, so no port can be probed and it
read "Not Deployed" while Portal worked (Workstation06, 2026-10-26). Pixel
Edge's /health answers ok only when that socket's /health does; an edge
that is down proves nothing about the socket.
"""
edge = SERVICES.get("pixel-edge")
if edge is None:
return _service_status_from_config(service_id, config, "not_deployed")
edge_status = (await check_service_health("pixel-edge", edge)).status
status = edge_status if edge_status in ("healthy", "not_deployed") else "unknown"
return _service_status_from_config(service_id, config, status)
async def _check_host_systemd_health(service_id: str, config: dict) -> ServiceStatus:
"""Check a host-managed service through the authenticated host-agent."""
if service_id == "opencode":
return await _check_opencode_health(service_id, config)
if service_id == "pixel-agent" and config.get("socket_only"):
return await _check_pixel_ingress_health(service_id, config)
return await _check_host_port_health(service_id, config)
async def _check_host_port_health(service_id: str, config: dict) -> ServiceStatus:
"""Check a host-managed service through the authenticated host-agent.
Host-systemd services such as OpenCode usually bind to host loopback. From
inside Docker, probing ``localhost`` checks the dashboard-api container
instead of the real host, so ask the host-agent to prove the local port is
open. If the proof is unavailable, fail closed so the dashboard does not
launch users into a dead localhost URL.
"""
port = int(config.get("health_port") or config.get("external_port") or config.get("port") or 0)
if port <= 0:
return _service_status_from_config(service_id, config, "not_deployed")
try:
payload = await request_agent_json(
"GET",
"/v1/host/port",
params={"host": "127.0.0.1", "port": port},
timeout=5,
)
except AgentClientError:
return _service_status_from_config(service_id, config, "down")
status = "healthy" if payload.get("reachable") else "not_deployed"
return ServiceStatus(
id=service_id,
name=config["name"],
port=config["port"],
external_port=config.get("external_port", config["port"]),
status=status,
response_time_ms=payload.get("response_time_ms"),
)
# --- Token Tracking ---
_TOKEN_FILE = Path(DATA_DIR) / "token_counter.json"
_PERF_FILE = Path(DATA_DIR) / "model_performance.json"
MAX_SINGLE_REQUEST_TOKENS_PER_SECOND = 10_000.0
_prev_tokens = {}
_llama_metrics_lock = None
_llama_metrics_sample = {}
_METRICS_SAMPLE_SECONDS = 1.0
_metrics_clock = time.monotonic
_metrics_wall_clock = time.time
_token_counter_lock = threading.Lock()
def _update_lifetime_tokens(server_counter: float, counter_id: Optional[str] = None) -> int:
"""Accumulate independent runtime/model counters without crossing baselines."""
with _token_counter_lock:
data = _read_json_file(_TOKEN_FILE, {})
if not isinstance(data, dict):
data = {}
current = _non_negative_number(server_counter)
prev = _non_negative_number(data.get("last_server_counter"))
if counter_id is not None:
counters = data.get("server_counters")
if not isinstance(counters, dict):
# Bind a legacy single baseline once, without recounting it.
counters = {counter_id: prev}
prev = _non_negative_number(counters.get(counter_id))
counters[counter_id] = current
data["server_counters"] = counters
lifetime = _non_negative_number(data.get("lifetime"))
delta = current if current < prev else current - prev
data["lifetime"] = int(lifetime + delta)
data["last_server_counter"] = current
_write_json_file(_TOKEN_FILE, data)
return data["lifetime"]
def _get_lifetime_tokens() -> int:
data = _read_json_file(_TOKEN_FILE, {})
if not isinstance(data, dict):
return 0
return int(_non_negative_number(data.get("lifetime")))
def _non_negative_number(value) -> float:
"""Return a finite non-negative number for persisted/runtime counters."""
try:
number = float(value)
except (TypeError, ValueError):
return 0.0
if not math.isfinite(number) or number < 0:
return 0.0
return number
def _normalize_perf_key(value: str | None) -> str:
return re.sub(r"[^a-z0-9]+", "-", str(value or "").lower()).strip("-")
def is_plausible_single_request_tps(value) -> bool:
"""Validate interactive single-request decode throughput, not batched capacity."""
try:
tokens_per_second = float(value)
except (TypeError, ValueError):
return False
return math.isfinite(tokens_per_second) and 0 < tokens_per_second <= MAX_SINGLE_REQUEST_TOKENS_PER_SECOND
def _read_json_file(path: Path, default):
try:
if path.exists():
return json.loads(path.read_text(encoding="utf-8"))
except (json.JSONDecodeError, OSError) as e:
logger.debug("Failed to read JSON file %s: %s", path, e)
return default
def _write_json_file(path: Path, data) -> None:
temporary = path.with_name(f".{path.name}.{os.getpid()}.{threading.get_ident()}.tmp")
try:
path.parent.mkdir(parents=True, exist_ok=True)
temporary.write_text(json.dumps(data, indent=2, sort_keys=True), encoding="utf-8")
# Windows virus scanners and indexers can briefly retain a handle to
# the destination. Retry only that transient access-denied case; other
# filesystem failures remain fail-soft as before.
for attempt in range(4):
try:
os.replace(temporary, path)
break
except PermissionError:
if attempt == 3:
raise
time.sleep(0.025 * (2 ** attempt))
except OSError as e:
logger.debug("Failed to write JSON file %s: %s", path, e)
finally:
try:
temporary.unlink(missing_ok=True)
except OSError:
pass
def _performance_key(backend: str, gpu_name: str, model_name: str,
context_length: Optional[int] = None,
gguf: Optional[str] = None,
vram_total_mb: Optional[int] = None) -> str:
parts = [
_normalize_perf_key(backend or "unknown"),
_normalize_perf_key(gpu_name),
_normalize_perf_key(model_name),
]
if gguf:
parts.append(_normalize_perf_key(gguf))
if context_length:
parts.append(f"ctx-{int(context_length)}")
if vram_total_mb:
parts.append(f"vram-{int(round(vram_total_mb / 1024))}gb")
return ":".join(parts)
def record_model_performance(
model_name: Optional[str],
gpu_name: Optional[str],
backend: str,
tokens_per_second: float,
*,
model_id: Optional[str] = None,
gguf: Optional[str] = None,
quantization: Optional[str] = None,
architecture: Optional[str] = None,
context_length: Optional[int] = None,
decode_read_mb: Optional[float] = None,
vram_total_mb: Optional[int] = None,
os_name: Optional[str] = None,
flags: Optional[dict] = None,
source: str = "local_metric",
) -> None:
"""Persist observed throughput for this exact machine/model pair."""
if not model_name or not gpu_name:
return
try:
tps = float(tokens_per_second)
except (TypeError, ValueError):
return
if not is_plausible_single_request_tps(tps):
logger.warning("Ignoring implausible single-request throughput sample: %s tok/s", tps)
return
data = _read_json_file(_PERF_FILE, {"schema_version": "ods.model-performance.v1", "samples": {}})
samples = data.setdefault("samples", {})
key = _performance_key(backend, gpu_name, model_name, context_length, gguf, vram_total_mb)
previous = samples.get(key, {})
previous_avg = float(previous.get("tokens_per_second", tps))
previous_count = int(previous.get("sample_count", 0))
if not is_plausible_single_request_tps(previous_avg):
previous_avg = tps
previous_count = 0
avg = (previous_avg * 0.8) + (tps * 0.2) if previous_count else tps
samples[key] = {
"model": model_name,
"model_id": model_id or previous.get("model_id"),
"gguf": gguf or previous.get("gguf"),
"quantization": quantization or previous.get("quantization"),
"architecture": architecture or previous.get("architecture"),
"gpu": gpu_name,
"backend": backend or "unknown",
"context_length": context_length or previous.get("context_length"),
"decode_read_mb": decode_read_mb or previous.get("decode_read_mb"),
"vram_total_mb": vram_total_mb or previous.get("vram_total_mb"),
"os": os_name or previous.get("os"),
"flags": flags or previous.get("flags", {}),
"source": source,
"tokens_per_second": round(avg, 1),
"last_tokens_per_second": round(tps, 1),
"sample_count": previous_count + 1,
"updated_at": int(time.time()),
}
samples[_performance_key(backend, gpu_name, model_name)] = samples[key]
_write_json_file(_PERF_FILE, data)
def get_recorded_model_performance(
model_name: str,
gpu_name: str,
backend: str,
*,
context_length: Optional[int] = None,
gguf: Optional[str] = None,
vram_total_mb: Optional[int] = None,
) -> Optional[dict]:
data = _read_json_file(_PERF_FILE, {"samples": {}})
keys = [
_performance_key(backend, gpu_name, model_name, context_length, gguf, vram_total_mb),
_performance_key(backend, gpu_name, model_name, context_length, gguf),
_performance_key(backend, gpu_name, model_name, context_length),
_performance_key(backend, gpu_name, model_name),
]
samples = data.get("samples", {})
sample = next((samples.get(k) for k in keys if samples.get(k)), None)
return sample if isinstance(sample, dict) else None
def get_model_performance_samples() -> list[dict]:
data = _read_json_file(_PERF_FILE, {"samples": {}})
samples = data.get("samples", {})
if not isinstance(samples, dict):
return []
return [sample for sample in samples.values() if isinstance(sample, dict)]
# --- LLM Metrics ---
def _measurement_number(value):
"""Keep missing/invalid observations distinct from a measured zero."""
if value is None or isinstance(value, bool):
return None
try:
number = float(value)
except (TypeError, ValueError):
return None
return number if math.isfinite(number) and number >= 0 else None
def _saved_lifetime_tokens():
data = _read_json_file(_TOKEN_FILE, {})
value = _measurement_number(data.get("lifetime")) if isinstance(data, dict) else None
return int(value) if value is not None else None
def _host_native_llm() -> bool:
"""The model runs in an ODS-owned llama-server on the Windows host.
That server requires its API key for /props and /metrics, which this
container never holds; the host agent reads them for the dashboard
(``/v1/llm/status``), through the WSL bridge when the stack runs in WSL.
"""
return (LLM_BACKEND != "external"
and read_live_env_value("AMD_INFERENCE_LOCATION").strip().lower() == "host")
async def _host_llm_status() -> dict:
status = await request_agent_json("GET", "/v1/llm/status", timeout=6)
if not isinstance(status, dict) or not isinstance(status.get("health"), dict):
raise ValueError("host llama-server status is invalid")
return status
def get_cached_llama_metrics() -> dict:
"""Last known measurement for status fallback; never claim fresh telemetry."""
result = dict(_llama_metrics_sample.get("result", {}))
result.update(throughput_state="unavailable", inference_active=None)
return result
async def get_llama_metrics(model_hint: Optional[str] = None) -> dict:
"""Share one measured sample across status/model consumers for one second.
Counter deltas describe generation intervals, not instantaneous token output.
Retain the last measured positive rate until another generation is observed,
with its original timestamp and explicit retained/unavailable provenance.
Counter updates may occur only on completion; request activity is separate.
Never compute a new interval across an outage/model change.
"""
global _llama_metrics_lock
if _llama_metrics_lock is None:
_llama_metrics_lock = asyncio.Lock()
model_name = model_hint
if model_name is None:
model_name = await get_loaded_model() or ""
service = SERVICES.get("llama-server", {})
identity = (LLM_BACKEND, service.get("host"), service.get("port"),
str(os.environ.get("LLAMA_METRICS_PORT", service.get("port", ""))), model_name,
"host-native" if _host_native_llm() else "")
async with _llama_metrics_lock:
now = _metrics_clock()
previous_identity = _llama_metrics_sample.get("identity")
if not model_name:
# Discovery failure is not proof of a different model. Hold only a
# same-endpoint historical sample, with its actual model identity.
same_endpoint = (previous_identity is not None
and previous_identity[:4] + previous_identity[5:] == identity[:4] + identity[5:])
previous = _llama_metrics_sample.get("measurement") if same_endpoint else None
_prev_tokens.clear()
if not same_endpoint:
_llama_metrics_sample.clear()
return {
"tokens_per_second": previous["rate"] if previous else None,
"lifetime_tokens": _saved_lifetime_tokens(),
"token_count_mode": "cumulative",
"throughput_mode": (previous.get("mode") if previous else None) or "generation_interval",
"throughput_state": "unavailable",
"throughput_sampled_at": previous["at"] if previous else None,
"throughput_model": previous_identity[4] if previous else None,
"inference_active": None,
}
if previous_identity != identity:
_prev_tokens.clear()
_llama_metrics_sample.clear()
elif now - _llama_metrics_sample["time"] < _METRICS_SAMPLE_SECONDS:
return dict(_llama_metrics_sample["result"])
counter_id = hashlib.sha256(json.dumps(identity).encode()).hexdigest()
try:
result = await _fetch_llama_metrics(model_hint=model_name, counter_id=counter_id)
except asyncio.CancelledError:
# A bounded background observer may time out while owning the
# sampler. Release the lock without measuring across that gap;
# cancellation while waiting for the lock never touches its owner.
_prev_tokens.clear()
if "result" in _llama_metrics_sample:
_llama_metrics_sample["result"].update(
throughput_state="unavailable", inference_active=None)
raise
mode = result.pop("_throughput_mode", "generation_interval")
available = result.pop("_available", False)
reset = result.pop("_counter_reset", False)
counters = result.pop("_counters", None)
previous_counters = _llama_metrics_sample.get("counters")
if counters is not None:
if previous_counters is not None:
reset = reset or any(old is not None and new is not None and new < old
for old, new in zip(previous_counters, counters))
_llama_metrics_sample["counters"] = counters
if reset:
_llama_metrics_sample.pop("measurement", None)
previous = _llama_metrics_sample.get("measurement")
completion_identity = result.pop("_completion_identity", None)
rate = result.get("tokens_per_second")
newly_measured = (available and rate is not None and rate > 0
and (completion_identity is None or previous is None
or completion_identity != previous.get("completion_identity")))
if newly_measured:
previous = {"rate": rate, "at": _metrics_wall_clock(), "mode": mode,
"completion_identity": completion_identity}
_llama_metrics_sample["measurement"] = previous
result["tokens_per_second"] = previous["rate"] if previous else None
result["throughput_sampled_at"] = previous["at"] if previous else None
result["throughput_state"] = ("unavailable" if not available or not previous
else "measured" if newly_measured else "retained")
result["throughput_mode"] = previous.get("mode", mode) if previous else mode
result.setdefault("inference_active", None)
result["throughput_model"] = model_name or None
_llama_metrics_sample.update(identity=identity, time=_metrics_clock(), result=dict(result))
return result
def _observe_live_output_slots(payload, sampled_at: float):
"""Rate of accepted output tokens for an unchanged set of active tasks.
llama.cpp b9014 server-context.cpp exports n_decoded as predicted_n and
increments it by accepted tokens, including accepted speculative tokens.
This is distinct from Prometheus n_decode_total (decode invocations).
Read only numeric identifiers/counters; never retain prompt/params/text.
The caller owns the shared sampler lock and clears this baseline on failure.
"""
if not isinstance(payload, list):
raise ValueError("slot metrics must be a list")
counts = {}
seen_slots = set()
for slot in payload:
if not isinstance(slot, dict) or not isinstance(slot.get("is_processing"), bool):
raise ValueError("invalid slot activity")
if not slot["is_processing"]:
continue
slot_id, task_id = slot.get("id"), slot.get("id_task")
next_token = slot.get("next_token")
# b9014 returns a one-element array; older servers return an object.
if isinstance(next_token, list) and len(next_token) == 1:
next_token = next_token[0]
if not isinstance(next_token, dict):
raise ValueError("slot output counter unavailable")
count = next_token.get("n_decoded")
if any(type(value) is not int or not 0 <= value < 2**63
for value in (slot_id, task_id, count)) or slot_id in seen_slots:
raise ValueError("invalid slot output counter or identity")
seen_slots.add(slot_id)
counts[(slot_id, task_id)] = count
previous = _prev_tokens.get("live_slots")
_prev_tokens["live_slots"] = {"at": sampled_at, "counts": counts}
if not counts or previous is None or previous["counts"].keys() != counts.keys():
return None
elapsed = sampled_at - previous["at"]
if elapsed <= 0 or any(count < previous["counts"][key] for key, count in counts.items()):
return None # task/reset/clock discontinuity starts a new interval
return round(sum(count - previous["counts"][key] for key, count in counts.items()) / elapsed, 1)
# vLLM's counters (LLM_BACKEND=vllm) under the names the sampler reads. vLLM
# exports no generation-seconds counter, so its throughput reads unavailable
# while its token count still accumulates.
_VLLM_COUNTERS = {
"vllm:generation_tokens_total": "tokens_predicted_total",
"vllm:num_requests_running": "requests_processing",
}
def _parse_llama_prometheus(body: str) -> dict:
"""Read the llama.cpp (or vLLM) counters the dashboard samples from a /metrics body."""
metrics: dict = {}
for line in body.split("\n"):
line = line.strip()
if not line or line.startswith("#"):
continue
parts = line.split()
if len(parts) < 2:
continue
metric_name = parts[0].split("{", 1)[0]
if metric_name in _VLLM_COUNTERS:
try:
metrics[_VLLM_COUNTERS[metric_name]] = float(parts[1])
except ValueError:
pass
continue
for counter in ("requests_processing", "tokens_predicted_total", "tokens_predicted_seconds_total"):
if metric_name.endswith(counter):
try:
metrics[counter] = float(parts[1])
except ValueError:
pass
return metrics
async def _fetch_llama_metrics(model_hint: Optional[str] = None, counter_id: Optional[str] = None) -> dict:
"""Get inference metrics from llama-server Prometheus /metrics endpoint.
Accepts an optional *model_hint* so callers that already resolved the
loaded model name can avoid a redundant HTTP round-trip.
"""
try:
slots_url = None
params: dict = {}
if _host_native_llm():
# The host agent already parsed llama.cpp's Prometheus counters.
reported = (await _host_llm_status()).get("metrics")
if not isinstance(reported, dict):
raise ValueError("host llama-server metrics are unavailable")
metrics = {key: reported[key] for key in (
"requests_processing", "tokens_predicted_total", "tokens_predicted_seconds_total",
) if key in reported}
client = None
else:
if "llama-server" not in SERVICES:
return {
"tokens_per_second": None,
"lifetime_tokens": _saved_lifetime_tokens(),
"token_count_mode": "cumulative",
}
host = SERVICES["llama-server"]["host"]
port = SERVICES["llama-server"]["port"]
metrics_port = int(os.environ.get("LLAMA_METRICS_PORT", port))
model_name = model_hint if model_hint is not None else (await get_loaded_model() or "")
url = f"http://{host}:{metrics_port}/metrics"
params = {"model": model_name} if model_name else {}
# /slots is llama.cpp's; vLLM has no per-slot output timing.
slots_url = None if LLM_BACKEND == "vllm" else f"http://{host}:{metrics_port}/slots"
client = await _get_httpx_client()
resp = await client.get(url, params=params)
resp.raise_for_status()
metrics = _parse_llama_prometheus(resp.text)
# A successful HTTP response is not sufficient proof that this is the
# llama.cpp Prometheus endpoint. Treat HTML, proxy error pages, and
# incomplete metric payloads as unavailable so they cannot reset the
# persistent server counter and double-count tokens after recovery.
if "tokens_predicted_total" not in metrics:
raise ValueError("llama-server metrics response has no token counter")
curr = _measurement_number(metrics["tokens_predicted_total"])
if curr is None:
raise ValueError("llama-server token counter is invalid")
gen_secs = _measurement_number(metrics.get("tokens_predicted_seconds_total"))
tps = None
reset = bool(_prev_tokens and (curr < _prev_tokens["count"] or
(gen_secs is not None and _prev_tokens.get("gen_secs") is not None
and gen_secs < _prev_tokens["gen_secs"])))
if _prev_tokens and gen_secs is not None:
delta_tokens = curr - _prev_tokens["count"]
previous_secs = _prev_tokens.get("gen_secs")
if previous_secs is not None:
delta_secs = gen_secs - previous_secs
if delta_tokens == 0 and delta_secs == 0:
tps = 0.0 # two successful unchanged observations
elif delta_tokens > 0 and delta_secs > 0:
tps = round(delta_tokens / delta_secs, 1)
# First samples, resets, or incomplete timing are unknown rates.
_prev_tokens.update(count=curr, gen_secs=gen_secs)
lifetime = _update_lifetime_tokens(curr, counter_id=counter_id)
active = (metrics["requests_processing"] > 0
if _measurement_number(metrics.get("requests_processing")) is not None else None)
available = gen_secs is not None
mode = "generation_interval"
if reset or active is not True:
_prev_tokens.pop("live_slots", None)
if active is True and slots_url:
try:
slots = await client.get(slots_url, params=params, timeout=2.0)
slots.raise_for_status()
live_rate = _observe_live_output_slots(slots.json(), _metrics_clock())
if live_rate is not None and live_rate > 0:
tps, available, mode = live_rate, True, "live_output_interval"
except (httpx.HTTPError, OSError, ValueError, KeyError):
_prev_tokens.pop("live_slots", None)
# A fresh completed interval remains valid if slots are disabled.
# Otherwise a held rate must expose the live telemetry outage.
available = bool(available and tps is not None and tps > 0)
return {
"tokens_per_second": tps,
"lifetime_tokens": lifetime,
"token_count_mode": "cumulative",
"_available": available,
"_throughput_mode": mode,
"_counter_reset": reset,
"_counters": (curr, gen_secs),
"inference_active": active,
}
except (AgentClientError, httpx.HTTPError, httpx.TimeoutException, OSError, ValueError, KeyError) as e:
_prev_tokens.clear() # never measure a rate across an unavailable gap
logger.warning("get_llama_metrics failed: %s: %s", type(e).__name__, e)
return {
"tokens_per_second": None,
"lifetime_tokens": _saved_lifetime_tokens(),
"token_count_mode": "cumulative",
}
async def get_loaded_model() -> Optional[str]:
"""Query llama-server for actually loaded model name."""
if _host_native_llm():
try:
health = (await _host_llm_status())["health"]
except (AgentClientError, OSError, ValueError):
return None
loaded = health.get("model_loaded")
return loaded.strip() if health.get("status") == "ok" and isinstance(loaded, str) and loaded.strip() else None
if "llama-server" not in SERVICES:
return None
try:
host = SERVICES["llama-server"]["host"]
port = SERVICES["llama-server"]["port"]
client = await _get_httpx_client()
# A generic OpenAI-compatible server lists every available model with
# no loaded status; its first entry is not the active model.
external_compatible = (
LLM_BACKEND == "external"
and os.environ.get("EXTERNAL_LLM_PROVIDER", "").strip().lower() == "openai-compatible"
)
# llama.cpp: /v1/models returns the loaded model with status info.
resp = await client.get(f"http://{host}:{port}{_LLM_API_PREFIX}/models")
models = resp.json().get("data", [])
for m in models:
status = m.get("status", {})
if isinstance(status, dict) and status.get("value") == "loaded":
return m.get("id")
if models and not external_compatible:
return models[0].get("id")
except (httpx.HTTPError, httpx.TimeoutException, ValueError, KeyError) as e:
logger.debug("get_loaded_model failed: %s", e)
return None
async def get_llama_context_size(model_hint: Optional[str] = None) -> Optional[int]:
"""Query llama-server /props for the actual n_ctx.
Accepts an optional *model_hint* to skip the redundant
``get_loaded_model()`` call when the caller already has it.
"""
if _host_native_llm():
try:
context = (await _host_llm_status())["health"].get("context_length")
except (AgentClientError, OSError, ValueError):
return None
return context if type(context) is int and context > 0 else None
if "llama-server" not in SERVICES:
return None
if LLM_BACKEND == "vllm":
return await _vllm_context_size(model_hint)
try:
host = SERVICES["llama-server"]["host"]
port = SERVICES["llama-server"]["port"]
loaded = model_hint if model_hint is not None else await get_loaded_model()
url = f"http://{host}:{port}/props"
if loaded:
url += f"?model={loaded}"
client = await _get_httpx_client()
resp = await client.get(url)
n_ctx = resp.json().get("default_generation_settings", {}).get("n_ctx")
return int(n_ctx) if n_ctx else None
except (httpx.HTTPError, httpx.TimeoutException, ValueError, KeyError) as e:
logger.debug("get_llama_context_size failed: %s", e)
return None
async def _vllm_context_size(model_hint: Optional[str]) -> Optional[int]:
"""The context vLLM serves: ``max_model_len`` of its /v1/models row.
vLLM has no /props; its --max-model-len fixes the context, and the row of
the served model reports it.
"""
try:
host = SERVICES["llama-server"]["host"]
port = SERVICES["llama-server"]["port"]
client = await _get_httpx_client()
resp = await client.get(f"http://{host}:{port}{_LLM_API_PREFIX}/models")
payload = resp.json()
except (httpx.HTTPError, httpx.TimeoutException, ValueError) as e:
logger.debug("vLLM context size unavailable: %s", e)
return None
rows = payload.get("data") if isinstance(payload, dict) else None
for row in rows if isinstance(rows, list) else ():
if isinstance(row, dict) and (model_hint is None or row.get("id") == model_hint):
context = row.get("max_model_len")
return context if type(context) is int and context > 0 else None
return None
async def get_llama_vision_support() -> Optional[bool]:
"""Whether the active llama-server loaded a vision projector.
llama-server reports it as ``/props`` ``modalities.vision``. None when that
cannot be read: the owner's own server, an unreachable runtime, a build
without the field, or the vLLM engine, which reports no modalities.
"""
if _host_native_llm():
try:
vision = (await _host_llm_status())["health"].get("vision")
except (AgentClientError, OSError, ValueError):
return None
return vision if type(vision) is bool else None
if LLM_BACKEND in {"external", "vllm"} or "llama-server" not in SERVICES:
return None
try:
host = SERVICES["llama-server"]["host"]
port = SERVICES["llama-server"]["port"]
client = await _get_httpx_client()
props = (await client.get(f"http://{host}:{port}/props")).json()
except (httpx.HTTPError, ValueError) as e:
logger.debug("get_llama_vision_support failed: %s", e)
return None
modalities = props.get("modalities") if isinstance(props, dict) else None
vision = modalities.get("vision") if isinstance(modalities, dict) else None
return vision if type(vision) is bool else None
# --- Service Health Cache ---
# Written by background poll loop in main.py, read by API endpoints.
# Keeps health checking decoupled from request handling so slow DNS
# lookups (Docker Desktop) never block API responses.
_services_cache: Optional[list] = None # list[ServiceStatus], set by poll loop
_services_cache_revision = 0
def _host_service_affirmed_stopped(service_id: str) -> bool:
"""True when the host agent confirmed an installed service is stopped.
Agent failures still surface as ``down``; those remain hidden for optional
host tools. A confirmed stopped OpenCode must stay visible so the owner can
start it instead of seeing it vanish or a permanent "Offline" entry.
"""
lifecycle = _opencode_lifecycle if service_id == "opencode" else None
return bool(lifecycle and lifecycle.get("state") == "stopped")
def _normalize_cached_service_status(status: ServiceStatus) -> ServiceStatus:
"""Avoid treating absent optional host-managed tools as broken services."""
config = SERVICES.get(status.id, {})
if (
status.status == "down"
and config.get("type") == "host-systemd"
and not config.get("required", False)
and not _host_service_affirmed_stopped(status.id)
):
return ServiceStatus(
id=status.id,
name=status.name,
port=status.port,
external_port=status.external_port,
status="not_deployed",
response_time_ms=status.response_time_ms,
)
return status
def get_services_cache_revision() -> int:
"""Revision captured before a background health poll starts."""
return _services_cache_revision
def set_services_cache(statuses: list, *, expected_revision: Optional[int] = None) -> bool:
"""Store a poll result only if no owner refresh superseded its snapshot."""
global _services_cache, _services_cache_revision
if expected_revision is not None and expected_revision != _services_cache_revision:
return False
_services_cache = [_normalize_cached_service_status(status) for status in statuses]
_services_cache_revision += 1
return True
def get_cached_services() -> Optional[list]:
"""Read cached health check results. Returns None if no poll has completed yet."""
return _services_cache
async def refresh_cached_service_status(service_id: str) -> None:
"""Re-check one service and replace its cached row after an owner action."""
global _services_cache, _services_cache_revision
config = SERVICES.get(service_id)
if config is None or _services_cache is None:
return
status = _normalize_cached_service_status(await check_service_health(service_id, config))
_services_cache = [status if item.id == service_id else item for item in _services_cache]
_services_cache_revision += 1
async def refresh_cached_builtin_services(service_ids: list[str]) -> None:
"""Publish current health after a Library start, before its response.
A newly selected built-in may be absent from both the import-time registry
and the cache. Use the same current manifests and Docker reconciliation as
the poller, and fence polls begun before or during this owner refresh.
The caller retains the affected extension operation locks throughout.
"""
global _services_cache, _services_cache_revision
selected = set(service_ids) & LIBRARY_MANAGEABLE_BUILTINS
if not selected or _services_cache is None:
return # An uncached catalog already performs a fresh full observation.
_services_cache_revision += 1
statuses = await get_all_services(only_service_ids=selected)
# Merge into the current list: an unrelated owner refresh may have finished
# while these probes awaited I/O. Never put its old row back.
_services_cache = [row for row in _services_cache if row.id not in selected]
_services_cache.extend(_normalize_cached_service_status(row) for row in statuses)
_services_cache_revision += 1
# --- Service Health ---
async def check_service_health(
service_id: str,
config: dict,
*,
timeout: Optional[aiohttp.ClientTimeout] = None,
) -> ServiceStatus:
"""Check if a service is healthy by hitting its health endpoint.
*timeout* overrides the session-level timeout for a single probe. The
catalog fan-out passes a shorter timeout so one slow service does not
stall the entire Extensions page.
"""
if config.get("type") == "host-systemd":
return await _check_host_systemd_health(service_id, config)
def port_number(value):
if type(value) is not int and not (
isinstance(value, str) and re.fullmatch(r"[0-9]{1,5}", value.strip())
):
raise ValueError("port must be an integer")
port = int(value)
if not 0 <= port <= 65535:
raise ValueError("port outside valid range")
return port
# Keep the service visible as down on malformed configuration, without
# probing an unrelated default port. Zero represents an invalid/absent port.
safe = {**config, "name": str(config.get("name") or service_id), "port": 0, "external_port": 0}
try:
safe["port"] = port_number(config.get("port"))
safe["external_port"] = port_number(config.get("external_port", safe["port"]))
except (ValueError, TypeError):
return _service_status_from_config(service_id, safe, "down")
config = safe
if config.get("host_network") and config["port"] == 0:
if service_id == "tailscale":
return await _check_tailscale_health(service_id, config)
return _service_status_from_config(service_id, config, "not_deployed")
host = config.get('host', 'localhost')
try:
health_path = config.get('health', '/')
if not isinstance(health_path, str):
raise ValueError("health path must be a string")
if not health_path.startswith('/'):
health_path = f"/{health_path}"
health_port = port_number(config.get('health_port', config['port']))
if health_port == 0:
raise ValueError("HTTP health port must be positive")
except (ValueError, TypeError):
return _service_status_from_config(service_id, config, "down")
# A model API (API mode) is not a local service: probe it with its own
# scheme and Host header. The probe carries no key (only LiteLLM holds
# it), so an API that answers 401/403 is up; this checks reachability.
external_api = config.get("external_api") is True
scheme = config.get("scheme") if external_api and config.get("scheme") in ("http", "https") else "http"
url = f"{scheme}://{host}:{health_port}{health_path}"
status = "unknown"
response_time = None
try:
session = await _get_aio_session()
start = asyncio.get_event_loop().time()
# Send Host header so reverse-proxy services (e.g. Caddy in Baserow)
# route the request correctly instead of returning 404. Some API
# front ends refuse a library User-Agent (Cloudflare error 1010).
headers = {"User-Agent": "ODS-Dashboard"} if external_api else {"Host": "localhost"}
get_kwargs: dict = {"headers": headers}
health_auth_env = config.get("health_auth_env")
if health_auth_env is not None:
prefix = service_id.upper().replace("-", "_") + "_"
if (not isinstance(health_auth_env, str)
or not re.fullmatch(r"[A-Z][A-Z0-9_]{1,127}", health_auth_env)
or not health_auth_env.startswith(prefix)
or host != service_id):
return _service_status_from_config(service_id, config, "unhealthy")
token = read_live_env_value(health_auth_env)
if not isinstance(token, str) or not token or len(token) > 8192 or any(ord(char) <= 32 or ord(char) >= 127 for char in token):
return _service_status_from_config(service_id, config, "unhealthy")
headers["Authorization"] = "Bearer " + token
# Never forward a local extension credential to a redirect target.
get_kwargs["allow_redirects"] = False
if timeout is not None:
get_kwargs["timeout"] = timeout
async with session.get(url, **get_kwargs) as resp:
response_time = (asyncio.get_event_loop().time() - start) * 1000
if external_api:
status = "healthy" if resp.status < 400 or resp.status in (401, 403) else "unhealthy"
else:
status = "healthy" if resp.status < (300 if health_auth_env is not None else 400) else "unhealthy"
except asyncio.TimeoutError:
# Service is reachable but slow — report degraded rather than down
# to avoid false "offline" flashes during startup or heavy load.
status = "degraded"
except aiohttp.ClientConnectorError as e:
if "Name or service not known" in str(e) or "nodename nor servname" in str(e):
status = "not_deployed"
else:
status = "down"
except (aiohttp.ClientError, OSError, ValueError) as e:
if config.get("health_auth_env") is None:
logger.debug(f"Health check failed for {service_id} at {url}: {e}")
else:
logger.debug("Authenticated health check failed for %s", service_id)
status = "down"
return ServiceStatus(
id=service_id, name=config["name"], port=config["port"],
external_port=config.get("external_port", config["port"]),
status=status, response_time_ms=round(response_time, 1) if response_time else None
)
def _switched_off_status(service_id: str, config: dict, **health_kwargs):
"""An awaitable status for Open WebUI when the install switched it off, else None.
The Compose resolver leaves Open WebUI out when ENABLE_OPEN_WEBUI is not
"true" (docker-compose.gateway-only.yml). Probing it then failed on name
resolution, and only two exact DNS error texts count as not deployed;
Docker in WSL words it differently, so Windows counted a core service
offline (fleet, Strixy: "6/7 core services online").
Where this API cannot read the owner's .env (an install owner other than
UID 1000), the flag is the one Compose passed when the container started.
A Library add changes the choice later, and Open WebUI then read "not
deployed" beside a healthy container (Workstation06, 2026-10-27), so the
host agent's reading of the owner's choice decides.
"""
if service_id != "open-webui":
return None
flag = read_live_env_value("ENABLE_OPEN_WEBUI")
env_unreadable = get_env_file_read_error() is not None
if (flag or "true").strip().lower() == "true":
return None
async def switched_off():
if env_unreadable and await _owner_selected_open_webui():
return await check_service_health(service_id, config, **health_kwargs)
return _service_status_from_config(service_id, config, "not_deployed")
return switched_off()
async def _owner_selected_open_webui() -> bool:
"""Whether the install owner's .env selects Open WebUI, as the host agent reads it."""
try:
selection = await request_agent_json("GET", "/v1/webui/selection", timeout=3)
except (AgentClientError, ValueError):
return False
return isinstance(selection, dict) and selection.get("enabled") is True
# Last time each service's own probe succeeded (monotonic seconds).
_last_probe_success: dict[str, float] = {}
# First completed observation bounds startup grace even if no probe succeeds.
_first_probe_seen: dict[str, float] = {}
_probe_clock = time.monotonic
_TIMEOUT_OVERRIDE_GRACE_S = 60.0
def _timeout_override_allowed(service_id: str, now: float) -> bool:
"""Bound Docker-only health to startup or a recent application success."""
last_ok = _last_probe_success.get(service_id)
anchor = last_ok if last_ok is not None else _first_probe_seen.get(service_id)
return anchor is not None and now - anchor <= _TIMEOUT_OVERRIDE_GRACE_S
_CRASH_LOOP_RESTARTS = 3
def container_crash_looping(item: dict) -> bool:
"""Whether a host service-health row shows a restart policy reviving a crash.
Each restart begins the container's health check anew, so a crash loop
reads as "starting" and its card said "installing" indefinitely
(Workstation06, 2026-10-27: Qdrant and TEI on ARM64). One or two
restarts can be a start-up race; a third is a loop.
"""
restarts = item.get("restart_count")
if type(restarts) is not int or restarts < _CRASH_LOOP_RESTARTS:
return False
state = str(item.get("state") or "").casefold()
return state == "restarting" or (
state == "running" and str(item.get("health") or "").casefold() == "starting")
async def get_all_services(*, only_service_ids: Optional[set[str]] = None) -> list[ServiceStatus]:
"""Get all service health statuses.
Uses ``return_exceptions=True`` so that one misbehaving service
cannot take down the entire status response.
"""
# The API can stay up while Library actions rename an optional built-in's
# Compose fragment. Refresh only qualified later-add services here so an
# omitted service becomes visible after Add, and disappears after Disable,
# without changing the import-time registry or probing every omitted app.
service_configs = dict(SERVICES)
try:
current_optional, _, _ = await asyncio.to_thread(
load_extension_manifests, EXTENSIONS_DIR, GPU_BACKEND,
only_service_ids=LIBRARY_MANAGEABLE_BUILTINS,
)
except OSError as exc:
logger.warning("Library built-in manifest refresh failed: %s", exc)
else:
for service_id in LIBRARY_MANAGEABLE_BUILTINS:
if service_id in current_optional:
service_configs.setdefault(service_id, current_optional[service_id])
else:
service_configs.pop(service_id, None)
if only_service_ids is not None:
service_configs = {sid: cfg for sid, cfg in service_configs.items() if sid in only_service_ids}
health_kwargs = {"timeout": _CATALOG_HEALTH_TIMEOUT} if only_service_ids is not None else {}
tasks = [_switched_off_status(sid, cfg, **health_kwargs) or check_service_health(sid, cfg, **health_kwargs)
for sid, cfg in service_configs.items()]
results = await asyncio.gather(*tasks, return_exceptions=True)
statuses: list[ServiceStatus] = []
for (sid, cfg), result in zip(service_configs.items(), results):
if isinstance(result, BaseException):
logger.warning("Health check for %s raised %s: %s", sid, type(result).__name__, result)
statuses.append(ServiceStatus(
id=sid, name=cfg["name"], port=cfg["port"],
external_port=cfg.get("external_port", cfg["port"]),
status="down", response_time_ms=None,
))
else:
statuses.append(result)
now = _probe_clock()
for status in statuses:
_first_probe_seen.setdefault(status.id, now)
if status.status == "healthy":
_last_probe_success[status.id] = now
if not any(status.status in {"degraded", "down", "unhealthy"} for status in statuses):
return statuses
try:
snapshot = await request_agent_json("GET", "/v1/service/health", timeout=15)
if snapshot.get("schema_version") != "ods.host-service-health.v1":
raise ValueError("unsupported host service-health schema")
containers = snapshot.get("containers")
if not isinstance(containers, list):
raise ValueError("host service-health containers must be a list")
by_service = {
str(item.get("service_id")): item
for item in containers
if isinstance(item, dict) and item.get("service_id")
}
by_name = {
str(item.get("container_name")): item
for item in containers
if isinstance(item, dict) and item.get("container_name")
}
except (AgentClientError, ValueError):
containers = []
by_service = {}
by_name = {}
reconciled: list[ServiceStatus] = []
for status in statuses:
config = service_configs.get(status.id, {})
item = by_service.get(status.id) or by_name.get(str(config.get("container_name") or ""))
replacement = status.status
startup_pending = False
crash_looping = False
if item and config.get("type", "docker") == "docker":
health = str(item.get("health") or "none").casefold()
state = str(item.get("state") or "unknown").casefold()
if container_crash_looping(item):
replacement = "unhealthy"
crash_looping = True
elif health == "healthy" and status.status == "degraded" and _timeout_override_allowed(status.id, now):
# A successful declared Docker healthcheck is authoritative for
# transient dashboard-network timeouts, but never masks HTTP
# errors or connection refusal from the application probe.
# The in-container check cannot see inter-container network
# loss, so it only covers startup or a short grace window.
replacement = "healthy"
elif health == "unhealthy":
replacement = "unhealthy"
elif state == "running" and health == "starting" and status.status in {"down", "degraded"}:
replacement = "degraded"
# Do not turn a timeout into startup guidance without one
# unambiguous running container's declared health state.
matches = [row for row in containers if isinstance(row, dict) and (
row.get("service_id") == status.id or (
config.get("container_name") and
row.get("container_name") == config["container_name"]
)
)]
startup_pending = len(matches) == 1
elif state in {"exited", "dead", "removing"}:
replacement = "down"
if (replacement != status.status or startup_pending != status.startup_pending
or crash_looping != status.crash_looping):
status = status.model_copy(update={"status": replacement, "startup_pending": startup_pending,
"crash_looping": crash_looping})
reconciled.append(status)
if _host_native_llm():
llama_index = next(
(index for index, status in enumerate(reconciled) if status.id == "llama-server"),
None,
)
if llama_index is not None and reconciled[llama_index].status != "healthy":
try:
health = (await _host_llm_status())["health"]
if str(health.get("status") or "").casefold() == "ok":
reconciled[llama_index] = reconciled[llama_index].model_copy(
update={"status": "healthy"},
)
except (AgentClientError, ValueError):
pass
return reconciled
# --- System Metrics ---
def dir_size_gb(path: Path) -> float:
"""Calculate total size of a directory in GB. Returns 0.0 if path doesn't exist.
Skips symlinks to avoid following links outside DATA_DIR and double-counting.
Results are cached for 60 seconds to avoid repeated expensive rglob walks.
"""
cached = _dir_size_cache.get(path)
if cached is not None:
return cached
if not path.exists():
_dir_size_cache.set(path, 0.0)
return 0.0
total = 0
try:
for f in path.rglob("*"):
try:
if f.is_symlink():
continue
if f.is_file():
total += f.stat().st_size
except (PermissionError, OSError):
pass
except (PermissionError, OSError):
pass
result = round(total / (1024**3), 2)
_dir_size_cache.set(path, result)
return result
def invalidate_dir_size_cache(path: Path):
"""Remove cached size for a specific path after it has been modified."""
_dir_size_cache.invalidate(path)
def clear_dir_size_cache():
"""Clear the entire dir_size_gb cache (e.g. after bulk operations)."""
_dir_size_cache.clear()
def get_disk_usage() -> DiskUsage:
"""Get disk usage for the ODS install directory."""
path = INSTALL_DIR if os.path.exists(INSTALL_DIR) else os.path.expanduser("~")
total, used, free = shutil.disk_usage(path)
percent = round(used / total * 100, 1) if total > 0 else 0.0
return DiskUsage(path=path, used_gb=round(used / (1024**3), 2), total_gb=round(total / (1024**3), 2), percent=percent)
def owner_model_config() -> Optional[dict]:
"""The configured model as the install owner's host agent reads it from .env.
For an install owner other than UID 1000, whose private .env this API
cannot read (Workstation06, 2026-10-26: nvuser, UID 1001). The agent
returns only validated identities. None when it cannot answer.
"""
try:
snapshot = request_agent_json_sync("GET", "/v1/model/config", timeout=5)
except AgentClientError:
return None
model = snapshot.get("configuredModel") if isinstance(snapshot, dict) else None
return model if isinstance(model, dict) else None
def get_model_info() -> Optional[ModelInfo]:
"""Get current model info from .env config."""
env_path = Path(INSTALL_DIR) / ".env"
if not env_path.exists():
return None
try:
env_values = {}
with open(env_path) as f:
for line in f:
if "=" not in line or line.lstrip().startswith("#"):
continue
key, value = line.split("=", 1)
key = key.strip()
if not key:
continue
value = parse_env_value(value)
env_values[key] = value
except PermissionError:
# An install owner other than UID 1000 keeps its private .env from this
# API: every status poll logged the refusal and the Dashboard showed no
# configured model. The owner's host agent reads it instead.
owner = owner_model_config() or {}
env_values = {"LLM_MODEL": owner.get("llmModel") or "", "GGUF_FILE": owner.get("ggufFile") or "",
"CTX_SIZE": str(owner.get("contextSize") or "")}
except OSError as e:
logger.warning("Failed to read .env for model info: %s", e)
return None
model_name = env_values.get("LLM_MODEL")
if not model_name:
return None
size_gb, quant = 15.0, None
# CTX_SIZE/MAX_CONTEXT come straight from .env and may be
# non-numeric (e.g. "auto", "8k", or a trailing comment); fall
# back to the default rather than 500-ing every caller. Mirrors
# the guard already used in routers/models.py.
context = 32768
for key in ("CTX_SIZE", "MAX_CONTEXT"):
try:
candidate = int(env_values.get(key) or 0)
except (TypeError, ValueError):
continue
if candidate > 0:
context = candidate
break
import re as _re
name_lower = model_name.lower()
if "gemma-4-e2b" in name_lower:
size_gb = 2.8
elif "gemma-4-e4b" in name_lower:
size_gb = 5.3
elif "gemma-4-26b" in name_lower:
size_gb = 18.0
elif "gemma-4-31b" in name_lower:
size_gb = 19.8
elif _re.search(r'\b2b\b', name_lower):
size_gb = 1.5
elif _re.search(r'\b4b\b', name_lower):
size_gb = 2.8
elif _re.search(r'\b7b\b', name_lower):
size_gb = 4.0
elif _re.search(r'\b8b\b', name_lower):
size_gb = 4.5
elif _re.search(r'\b9b\b', name_lower):
size_gb = 5.8
elif _re.search(r'\b14b\b', name_lower):
size_gb = 8.0
elif _re.search(r'\b26b\b', name_lower):
size_gb = 18.0
elif _re.search(r'\b30b\b', name_lower):
size_gb = 18.6
elif _re.search(r'\b31b\b', name_lower):
size_gb = 19.8
elif _re.search(r'\b32b\b', name_lower):
size_gb = 16.0
elif _re.search(r'\b70b\b', name_lower):
size_gb = 35.0
gguf_file = env_values.get("GGUF_FILE", "").lower()
if "awq" in name_lower:
quant = "AWQ"
elif "gptq" in name_lower:
quant = "GPTQ"
elif "gguf" in name_lower or gguf_file.endswith(".gguf"):
quant = "GGUF"
return ModelInfo(name=model_name, size_gb=size_gb, context_length=context, quantization=quant)
def get_bootstrap_status() -> BootstrapStatus:
"""Get bootstrap download progress if active."""
status_file = Path(DATA_DIR) / "bootstrap-status.json"
if not status_file.exists():
return BootstrapStatus(active=False)
try:
with open(status_file) as f:
data = json.load(f)
status = data.get("status", "")
if status in ("complete", "failed", "cancelled", "error"):
return BootstrapStatus(active=False)
if status == "" and not data.get("bytesDownloaded") and not data.get("percent"):
return BootstrapStatus(active=False)
phase = status if status in ("starting", "downloading", "verifying", "swapping") else None
# Reconcile with the filesystem only for non-active states. If the
# target model file is already present on disk and the status is
# non-active, the download is done enough for UI purposes. Active
# states remain busy because config updates and the llama-server
# hot-swap may not have finished yet; returning inactive here would
# hide a subsequent failure.
model_name = data.get("model")
if model_name and status not in ("downloading", "verifying", "swapping"):
models_dir = Path(DATA_DIR) / "models"
model_path = (models_dir / model_name).resolve()
if model_path.is_relative_to(models_dir.resolve()):
try:
if model_path.exists() and model_path.stat().st_size > 0:
return BootstrapStatus(active=False)
except OSError as e:
logger.debug("bootstrap reconciliation stat failed: %s", e)
eta_str = data.get("eta", "")
eta_seconds = None
if eta_str and eta_str.strip() and eta_str.strip() != "calculating...":
try:
parts = [p.strip() for p in eta_str.replace("m", "").replace("s", "").split() if p.strip()]
if len(parts) == 2:
eta_seconds = int(parts[0]) * 60 + int(parts[1])
elif len(parts) == 1:
eta_seconds = int(parts[0])
except (ValueError, IndexError):
pass
bytes_downloaded = data.get("bytesDownloaded", 0)
bytes_total = data.get("bytesTotal", 0)
speed_bps = data.get("speedBytesPerSec", 0)
percent_raw = data.get("percent")
percent = None
if percent_raw is not None:
try:
percent = max(0.0, min(100.0, float(percent_raw)))
except (ValueError, TypeError):
pass
if bytes_total and bytes_downloaded:
bytes_downloaded = max(0, min(bytes_downloaded, bytes_total))
return BootstrapStatus(
active=True, phase=phase, model_name=data.get("model"), percent=percent,
downloaded_gb=bytes_downloaded / (1024**3) if bytes_downloaded else None,
total_gb=bytes_total / (1024**3) if bytes_total else None,
speed_mbps=speed_bps / (1024**2) if speed_bps else None,
eta_seconds=eta_seconds
)
except (json.JSONDecodeError, OSError, KeyError) as e:
logger.warning("Failed to parse bootstrap status: %s", e)
return BootstrapStatus(active=False)
def get_uptime() -> int:
"""Get system uptime in seconds (cross-platform)."""
_system = platform.system()
import subprocess
try:
if _system == "Linux":
with open("/proc/uptime") as f:
return int(float(f.read().split()[0]))
elif _system == "Darwin":
result = subprocess.run(
["sysctl", "-n", "kern.boottime"],
capture_output=True, text=True, timeout=5,
)
if result.returncode == 0:
# Output: "{ sec = 1234567890, usec = 0 } ..."
import re
match = re.search(r"sec\s*=\s*(\d+)", result.stdout)
if match:
import time as _time
return int(_time.time()) - int(match.group(1))
elif _system == "Windows":
import ctypes
return ctypes.windll.kernel32.GetTickCount64() // 1000
except (OSError, subprocess.SubprocessError, ValueError, IndexError, AttributeError) as e:
logger.debug("get_uptime failed on %s: %s", _system, e)
return 0
def _get_cpu_metrics_linux() -> dict:
"""Get CPU usage from /proc/stat (Linux only)."""
result = {"percent": None, "temp_c": None}
try:
with open("/proc/stat") as f:
line = f.readline()
parts = line.split()
if len(parts) >= 8:
idle = int(parts[4]) + int(parts[5])
total = sum(int(p) for p in parts[1:8])
if not hasattr(get_cpu_metrics, "_prev"):
get_cpu_metrics._prev = (idle, total)
prev_idle, prev_total = get_cpu_metrics._prev
d_idle, d_total = idle - prev_idle, total - prev_total
get_cpu_metrics._prev = (idle, total)
if d_total > 0:
result["percent"] = max(0.0, min(100.0, round((1 - d_idle / d_total) * 100, 1)))
except (OSError, ValueError) as e:
logger.debug("Failed to read /proc/stat: %s", e)
try:
import glob
for tz in sorted(glob.glob("/sys/class/thermal/thermal_zone*/type")):
try:
with open(tz) as f:
zone_type = f.read().strip()
if any(k in zone_type.lower() for k in ("k10temp", "coretemp", "cpu", "soc", "tctl")):
with open(tz.replace("/type", "/temp")) as f:
result["temp_c"] = int(f.read().strip()) // 1000
break
except (OSError, ValueError):
continue
if result["temp_c"] is None:
for hwmon in sorted(glob.glob("/sys/class/hwmon/hwmon*/name")):
try:
with open(hwmon) as f:
name = f.read().strip()
if name in ("k10temp", "coretemp", "zenpower"):
with open(hwmon.replace("/name", "/temp1_input")) as f:
result["temp_c"] = int(f.read().strip()) // 1000
break
except (OSError, ValueError):
continue
except OSError as e:
logger.debug("Failed to read CPU temperature: %s", e)
return result
def _get_cpu_metrics_darwin() -> dict:
"""Get CPU usage on macOS via host_processor_info."""
result = {"percent": None, "temp_c": None}
try:
import subprocess
out = subprocess.run(
["top", "-l", "1", "-n", "0", "-stats", "cpu"],
capture_output=True, text=True, timeout=5,
)
if out.returncode == 0:
import re
match = re.search(r"CPU usage:\s+([\d.]+)%\s+user.*?([\d.]+)%\s+sys", out.stdout)
if match:
result["percent"] = round(float(match.group(1)) + float(match.group(2)), 1)
except (subprocess.SubprocessError, OSError, ValueError) as e:
logger.debug("macOS CPU metrics failed: %s", e)
return result
def get_cpu_metrics() -> dict:
"""Get CPU usage percentage and temperature (cross-platform)."""
_system = platform.system()
if _system == "Linux":
if os.environ.get("GPU_BACKEND", "").lower() == "apple":
return apple_host_metrics()["cpu"]
if linux_scope() == "wsl":
native = windows_host_metrics()["cpu"]
if native is not None:
return native
return {**_get_cpu_metrics_linux(), "scope": linux_scope(), "source": "linux-procfs"}
elif _system == "Darwin":
return {**_get_cpu_metrics_darwin(), "scope": "host", "source": "macos-top"}
return {"percent": None, "temp_c": None}
def _get_ram_metrics_linux() -> dict:
"""Get RAM usage from /proc/meminfo (Linux only)."""
result = {"used_gb": None, "total_gb": None, "percent": None}
try:
meminfo = {}
with open("/proc/meminfo") as f:
for line in f:
parts = line.split()
if len(parts) >= 2:
meminfo[parts[0].rstrip(":")] = int(parts[1])
total = meminfo.get("MemTotal", 0)
if total <= 0 or "MemAvailable" not in meminfo:
return result
available = meminfo["MemAvailable"]
used = max(0, total - available)
result["total_gb"] = round(total / (1024 * 1024), 1)
result["used_gb"] = round(used / (1024 * 1024), 1)
if total > 0:
result["percent"] = max(0.0, min(100.0, round(used / total * 100, 1)))
except (OSError, ValueError) as e:
logger.debug("Failed to read /proc/meminfo: %s", e)
return result
def _get_ram_metrics_sysctl() -> dict:
"""Get RAM usage on macOS via sysctl."""
result = {"used_gb": None, "total_gb": None, "percent": None}
try:
import subprocess
out = subprocess.run(
["sysctl", "-n", "hw.memsize"],
capture_output=True, text=True, timeout=5,
)
if out.returncode == 0:
total_bytes = int(out.stdout.strip())
if total_bytes <= 0:
return result
total_gb = total_bytes / (1024 ** 3)
result["total_gb"] = round(total_gb, 1)
# vm_stat for used memory
vm = subprocess.run(
["vm_stat"], capture_output=True, text=True, timeout=5,
)
if vm.returncode == 0:
import re
pages = {}
for line in vm.stdout.splitlines():
match = re.match(r"(.+?):\s+(\d+)", line)
if match:
pages[match.group(1).strip()] = int(match.group(2))
page_size = None
ps_match = re.search(r"page size of (\d+) bytes", vm.stdout)
if ps_match:
page_size = int(ps_match.group(1))
if page_size is None or not all(key in pages for key in ("Pages active", "Pages wired down", "Pages occupied by compressor")):
return result
active = pages["Pages active"]
wired = pages.get("Pages wired down", 0)
compressed = pages.get("Pages occupied by compressor", 0)
used_bytes = (active + wired + compressed) * page_size
result["used_gb"] = round(used_bytes / (1024 ** 3), 1)
if total_bytes > 0:
result["percent"] = round(used_bytes / total_bytes * 100, 1)
except (subprocess.SubprocessError, OSError, ValueError) as e:
logger.debug("macOS RAM metrics failed: %s", e)
return result
def get_ram_metrics() -> dict:
"""Get RAM usage (cross-platform)."""
_system = platform.system()
if _system == "Linux":
if os.environ.get("GPU_BACKEND", "").lower() == "apple":
return apple_host_metrics()["ram"]
if linux_scope() == "wsl":
native = windows_host_metrics()["ram"]
if native is not None:
return native
return {**_get_ram_metrics_linux(), "scope": linux_scope(), "source": "linux-procfs"}
elif _system == "Darwin":
return {**_get_ram_metrics_sysctl(), "scope": "host", "source": "macos-vm-stat"}
return {"used_gb": None, "total_gb": None, "percent": None}
def string_extract_domain_names_safe(text: str) -> list:
"""
Safely extract domain names (hostnames) from a raw string or text block.
Guards against None, non-string input, empty string, malformed URLs, and regex exceptions.
Returns a sorted list of unique lowercase domain names.
"""
if not isinstance(text, str) or not text.strip():
return []
if len(text) > 65536:
return []
# Tokenize first so an invalid long label cannot match a valid suffix.
# This extracts text candidates; it is not an SSRF/URL authorization check.
domains = set()
for candidate in re.findall(r"[A-Za-z0-9.-]+", text):
candidate = candidate.lower().strip(".")
labels = candidate.split(".")
if (len(candidate) <= 253 and len(labels) >= 2
and 2 <= len(labels[-1]) <= 63 and labels[-1].isalpha()
and all(re.fullmatch(r"[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?", label)
for label in labels)):
domains.add(candidate)
return sorted(domains)
def dict_key_path_setter_safe(d: dict, path_keys: list, value: any) -> dict:
"""
Safely set a nested key value in a dictionary given a list of path keys.
Guards against invalid dictionaries, paths over 128 keys, and unsupported keys.
Invalid paths leave the dictionary untouched; valid paths replace scalar parents.
Returns the modified dictionary (or a new dict if d is None/invalid).
"""
if d is None or not isinstance(d, dict):
d = {}
if (not isinstance(path_keys, (list, tuple)) or not 1 <= len(path_keys) <= 128
or any(type(key) not in (str, int) for key in path_keys)):
return d
current = d
for key in path_keys[:-1]:
k_str = key
if k_str not in current or not isinstance(current[k_str], dict):
current[k_str] = {}
current = current[k_str]
final_key = path_keys[-1]
current[final_key] = value
return d
def numeric_safe_geometric_mean(numbers: list) -> float:
"""
Safely compute the geometric mean of a list of positive numbers.
Guards against None, empty list, non-sequence types, negative/zero numbers,
NaN/Inf values, and float overflow/underflow using log-sum.
"""
if not isinstance(numbers, (list, tuple)) or not numbers:
return 0.0
import math
valid_nums = []
for x in numbers:
if isinstance(x, (int, float)) and not isinstance(x, bool):
try:
number = float(x)
except (OverflowError, ValueError):
continue
if math.isfinite(number) and number > 0:
valid_nums.append(number)
if not valid_nums:
return 0.0
try:
scale = max(valid_nums)
smallest = min(valid_nums)
if smallest == scale:
return scale
mean_log = math.fsum(math.log(x) / len(valid_nums) for x in valid_nums)
return min(scale, max(smallest, math.exp(mean_log)))
except OverflowError:
return scale # Rounding at the largest representable finite float.
except ValueError:
return 0.0
def list_deduplicate_by_key_safe(items: list, key_or_attr: any) -> list:
"""
Safely deduplicate a list of dictionaries or objects by a specified key or attribute,
preserving original order and guarding against None, unhashable keys, type errors, or missing keys.
"""
if not isinstance(items, (list, tuple)):
return []
if key_or_attr is None or not isinstance(key_or_attr, (str, int)):
return list(items)
seen = set()
structured = []
missing = object()
result = []
for item in items:
val = missing
if isinstance(item, dict):
val = item.get(key_or_attr, missing)
else:
try:
val = getattr(item, str(key_or_attr), missing)
except (AttributeError, TypeError, ValueError):
val = missing
if val is missing:
result.append(item)
continue
try:
key_val = (type(val), val)
hash(key_val)
except TypeError:
# Do not equate a list with its string representation or discard
# unrelated records that have no key.
try:
duplicate = any(type(val) is type(previous) and val == previous for previous in structured)
except (TypeError, ValueError, RecursionError):
duplicate = False
if not duplicate:
structured.append(val)
result.append(item)
continue
if key_val not in seen:
seen.add(key_val)
result.append(item)
return result
def string_snake_to_pascal_case_safe(text: str) -> str:
"""
Safely convert snake_case or kebab-case string to PascalCase.
Guards against None, non-string, whitespace, numbers, and multiple delimiters.
"""
if not isinstance(text, str) or not text.strip():
return ""
import re
clean = text.strip().replace("-", "_")
parts = [p for p in re.split(r'_+', clean) if p]
if not parts:
return ""
return "".join(p.capitalize() for p in parts)
def dict_flatten_nested_safe(d: dict, separator: str = '.', max_depth: int = 10) -> dict:
"""
Safely flatten a nested dictionary into a flat dictionary with delimiter-separated keys.
Guards against None, non-dict, maximum recursion depth limit, circular references, and non-string separators.
"""
if not isinstance(d, dict):
return {}
if not isinstance(separator, str):
separator = '.'
if type(max_depth) is not int or max_depth < 1:
max_depth = 10
max_depth = min(max_depth, 128)
result = {}
# Iterative traversal avoids Python recursion limits. Cycles and depth
# boundaries remain leaf values, just like other unflattened dictionaries.
pending = [(d, '', 0, frozenset({id(d)}))]
while pending:
current, prefix, depth, ancestors = pending.pop()
for k, v in current.items():
str_key = str(k)
new_key = f"{prefix}{separator}{str_key}" if prefix else str_key
if isinstance(v, dict) and v and depth + 1 < max_depth and id(v) not in ancestors:
pending.append((v, new_key, depth + 1, ancestors | {id(v)}))
else:
result[new_key] = v
return result
def numeric_exponential_moving_average_safe(values: list, alpha: float = 0.2) -> list:
"""
Safely compute Exponential Moving Average (EMA) over a numeric sequence.
Guards against None, non-sequence types, empty lists, NaN/Inf floats, and invalid alpha range (0 < alpha <= 1).
"""
if not isinstance(values, (list, tuple)) or not values:
return []
import math
if not isinstance(alpha, (int, float)) or isinstance(alpha, bool) or not 0 < alpha <= 1:
alpha = 0.2
if alpha <= 0 or alpha > 1:
alpha = 0.2
valid_vals = []
for v in values:
if isinstance(v, (int, float)) and not isinstance(v, bool):
try:
number = float(v)
except (OverflowError, ValueError):
continue
if math.isfinite(number):
valid_vals.append(number)
if not valid_vals:
return []
ema = []
current = valid_vals[0]
ema.append(current)
for v in valid_vals[1:]:
current = alpha * v + (1 - alpha) * current
ema.append(current)
return ema