Skip to content

Dodaios is an open-source agent OS for your desktop: it runs your coding agents, browser and apps by voice or chat, on your own machine. Formerly Personal Jarvis.

feat(local-models): six-hourly self-check with a badge, mounted assistant routes

rubenluetke10-beep18d482646d88
diff --git a/jarvis/core/config.py b/jarvis/core/config.py
index 65aae0cb0f..d538305492 100644
--- a/jarvis/core/config.py
+++ b/jarvis/core/config.py
@@ -863,6 +863,9 @@ class BrainProviderConfig(BaseModel):
# install that wants no outbound Hugging Face traffic makes none; read via
# :func:`ollama_hf_enabled` by the local-models routes.
hf_enabled: bool = False
+ # Cadence of the Local models self-check (badge only, no toasts), in
+ # hours; read by :func:`jarvis.local_models.health_monitor.interval_hours`.
+ health_check_hours: float = 6.0
@field_validator("models", mode="before")
@classmethod
diff --git a/jarvis/local_models/health_monitor.py b/jarvis/local_models/health_monitor.py
new file mode 100644
index 0000000000..e6f0676e97
--- /dev/null
+++ b/jarvis/local_models/health_monitor.py
@@ -0,0 +1,243 @@
+"""A bounded, quiet self-check of the local setup — a badge, never a nag.
+
+Started from the web server's lifespan next to the marketplace refresh
+scheduler: first run ten minutes after boot, then every
+``[brain.providers.ollama].health_check_hours`` (default 6). Nothing here
+runs on the boot path (AP-26) and nothing toasts: the result is one record
+in ``DATA_DIR/state/local_models_health.json`` (the same file the setup test
+writes), which the sidebar badge and ``GET /api/providers/section-health``
+read.
+
+One check: skip entirely when the server is down AND no role is configured
+(nothing to watch yet); otherwise ``probe_host`` plus one real 1-token chat
+generation on the configured chat model, capped at 30 s. The clock, the
+sleeper and the probes are injectable so the schedule is testable in
+milliseconds with fakes.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import datetime as _dt
+import logging
+from collections.abc import Awaitable, Callable
+from typing import Any
+
+log = logging.getLogger(__name__)
+
+__all__ = [
+ "DEFAULT_INTERVAL_HOURS",
+ "FIRST_RUN_DELAY_S",
+ "GENERATION_CAP_S",
+ "HealthMonitor",
+ "check_once",
+ "interval_hours",
+ "read_health_record",
+ "write_health_record",
+]
+
+FIRST_RUN_DELAY_S = 600.0
+DEFAULT_INTERVAL_HOURS = 6.0
+GENERATION_CAP_S = 30.0
+#: Never spin faster than this, whatever the config says.
+MIN_INTERVAL_S = 60.0
+
+ProbeFn = Callable[[str], Awaitable[dict[str, Any]]]
+GenerateFn = Callable[[Any, str], Awaitable[Any]]
+
+
+# ── config + record ───────────────────────────────────────────────────────
+
+
+def interval_hours(cfg: Any) -> float:
+ """``[brain.providers.ollama].health_check_hours`` — the ONE reader (AP-31)."""
+ providers = getattr(getattr(cfg, "brain", None), "providers", None) or {}
+ ollama = providers.get("ollama") if isinstance(providers, dict) else None
+ try:
+ hours = float(getattr(ollama, "health_check_hours", DEFAULT_INTERVAL_HOURS))
+ except (TypeError, ValueError):
+ return DEFAULT_INTERVAL_HOURS
+ return hours if hours > 0 else DEFAULT_INTERVAL_HOURS
+
+
+def read_health_record() -> dict[str, Any]:
+ """The badge record: ``{status, reason, since, last_ok, checked_at}`` (file only)."""
+ from jarvis.local_models.assistant_test import load_last_report
+
+ payload = load_last_report() or {}
+ return {
+ "status": str(payload.get("status") or "unknown"),
+ "reason": str(payload.get("reason") or ""),
+ "since": payload.get("since"),
+ "last_ok": payload.get("last_ok"),
+ "checked_at": payload.get("checked_at"),
+ }
+
+
+def write_health_record(status: str, reason: str, *, checked_at: str | None = None) -> bool:
+ """Merge the badge keys into the health file, keeping the last test's table."""
+ from jarvis.local_models.assistant_test import (
+ _atomic_write_json,
+ health_path,
+ load_last_report,
+ )
+
+ stamp = checked_at or _dt.datetime.now(_dt.UTC).isoformat(timespec="seconds")
+ previous = load_last_report() or {}
+ payload = dict(previous)
+ payload["status"] = status
+ payload["reason"] = reason
+ payload["checked_at"] = stamp
+ payload["since"] = (
+ previous.get("since")
+ if previous.get("status") == status and previous.get("since")
+ else stamp
+ )
+ payload["last_ok"] = stamp if status == "ok" else previous.get("last_ok")
+ payload["monitor"] = {"checked_at": stamp, "status": status, "reason": reason}
+ return _atomic_write_json(health_path(), payload)
+
+
+# ── one check ─────────────────────────────────────────────────────────────
+
+
+def _configured_roles(cfg: Any) -> dict[str, str]:
+ from jarvis.brain.ollama_roles import WRITABLE_ROLE_IDS, current_pick
+
+ out: dict[str, str] = {}
+ for role in WRITABLE_ROLE_IDS:
+ try:
+ model, _note = current_pick(cfg, role)
+ except Exception: # noqa: BLE001 — a broken pick is "not configured" for the badge
+ log.debug("health monitor: pick of %s unreadable", role, exc_info=True)
+ continue
+ if model:
+ out[role] = model
+ return out
+
+
+async def _default_probe(root: str) -> dict[str, Any]:
+ from jarvis.brain.ollama_runtime import probe_host
+
+ return await probe_host(root)
+
+
+async def _default_generate(cfg: Any, model: str) -> Any:
+ from jarvis.brain import provider_test
+ from jarvis.local_models.assistant_test import _OllamaSpec
+
+ return await provider_test.run_provider_test(
+ _OllamaSpec(), cfg, model=model, timeout_s=GENERATION_CAP_S
+ )
+
+
+async def check_once(
+ cfg: Any,
+ *,
+ root: str | None = None,
+ probe: ProbeFn | None = None,
+ generate: GenerateFn | None = None,
+ persist: bool = True,
+) -> dict[str, Any] | None:
+ """One self-check; returns the record written, or ``None`` when skipped."""
+ from jarvis.local_models.assistant_test import _server_root
+
+ root = root or _server_root(cfg)
+ roles = _configured_roles(cfg)
+ server = await (probe or _default_probe)(root)
+ if not server.get("ok") and not roles:
+ log.debug("health monitor: server down and nothing configured — skipped")
+ return None
+ if not server.get("ok"):
+ status, reason = "error", str(server.get("detail") or "Ollama did not answer.")
+ elif not roles:
+ status, reason = "needs_setup", "Ollama runs, but no role is configured."
+ else:
+ model = roles.get("chat") or roles.get("deep") or roles.get("tools_screen") or ""
+ if not model:
+ status, reason = "ok", ""
+ else:
+ try:
+ result = await asyncio.wait_for(
+ (generate or _default_generate)(cfg, model), timeout=GENERATION_CAP_S
+ )
+ except TimeoutError:
+ result = None
+ status, reason = "error", f"{model} did not answer within {GENERATION_CAP_S:.0f} s."
+ except Exception as exc: # noqa: BLE001 — the badge says so, the log keeps it
+ log.info("health monitor: generation on %s failed", model, exc_info=True)
+ result = None
+ status, reason = "error", f"{model}: {type(exc).__name__}: {exc}"
+ if result is not None:
+ if getattr(result, "status", "") == "ok":
+ status, reason = "ok", ""
+ else:
+ detail = getattr(result, "detail", "") or getattr(result, "status", "error")
+ status, reason = "error", f"{model}: {detail}"
+ record = {"status": status, "reason": reason}
+ if persist:
+ write_health_record(status, reason)
+ return record
+
+
+# ── the schedule ──────────────────────────────────────────────────────────
+
+
+class HealthMonitor:
+ """The periodic task: ``start()`` once after boot, ``stop()`` on shutdown."""
+
+ def __init__(
+ self,
+ cfg_fn: Callable[[], Any],
+ *,
+ first_delay_s: float = FIRST_RUN_DELAY_S,
+ sleep: Callable[[float], Awaitable[None]] | None = None,
+ check: Callable[[Any], Awaitable[Any]] | None = None,
+ ) -> None:
+ self._cfg_fn = cfg_fn
+ self._first_delay_s = first_delay_s
+ self._sleep = sleep or asyncio.sleep
+ self._check = check or check_once
+ self._task: asyncio.Task[None] | None = None
+ self.runs = 0
+
+ @property
+ def running(self) -> bool:
+ return self._task is not None and not self._task.done()
+
+ def start(self) -> None:
+ if self.running:
+ return
+ self._task = asyncio.create_task(self._loop(), name="local-models-health-monitor")
+
+ async def stop(self) -> None:
+ task = self._task
+ self._task = None
+ if task is None or task.done():
+ return
+ task.cancel()
+ try:
+ await task
+ except asyncio.CancelledError:
+ pass
+ except Exception: # noqa: BLE001 — shutdown stays best-effort
+ log.debug("health monitor: task ended with an error on stop", exc_info=True)
+
+ def _interval_s(self) -> float:
+ try:
+ return max(MIN_INTERVAL_S, interval_hours(self._cfg_fn()) * 3600.0)
+ except Exception: # noqa: BLE001 — a broken config keeps the default cadence
+ log.debug("health monitor: interval unreadable, using the default", exc_info=True)
+ return DEFAULT_INTERVAL_HOURS * 3600.0
+
+ async def _loop(self) -> None:
+ await self._sleep(self._first_delay_s)
+ while True:
+ try:
+ await self._check(self._cfg_fn())
+ except asyncio.CancelledError:
+ raise
+ except Exception: # noqa: BLE001 — one failed check never ends the schedule
+ log.warning("health monitor: check failed", exc_info=True)
+ self.runs += 1
+ await self._sleep(self._interval_s())
diff --git a/jarvis/ui/web/provider_routes.py b/jarvis/ui/web/provider_routes.py
index 304aed66e1..588fa33d01 100644
--- a/jarvis/ui/web/provider_routes.py
+++ b/jarvis/ui/web/provider_routes.py
@@ -2002,9 +2002,30 @@ async def _compute_section_health(
*(_safe_check(section, check) for section, check in checks.items())
)
sections.update(zip(checks, results, strict=True))
+ # The Local models sidebar badge: the persisted self-check record only —
+ # a file read, never a live call from this polled route.
+ sections["local_models"] = await _safe_check(
+ "local_models", asyncio.to_thread(_local_models_section_health)
+ )
return sections
+def _local_models_section_health() -> SectionHealth:
+ """The Local models badge from ``state/local_models_health.json`` (file only)."""
+ from jarvis.local_models.health_monitor import read_health_record
+
+ record = read_health_record()
+ status = record["status"]
+ if status not in _section_health.SECTION_HEALTH_STATUSES:
+ status = _section_health.UNKNOWN
+ return SectionHealth(
+ status=status,
+ reason=record["reason"] or status,
+ detail=record["reason"] or "Local models: no self-check yet",
+ subject_id="ollama",
+ )
+
+
# ----------------------------------------------------------------------
# Per-provider model picker (live catalog + pin + honest probe)
# ----------------------------------------------------------------------
diff --git a/jarvis/ui/web/server.py b/jarvis/ui/web/server.py
index d5ab6d0889..013d1f8009 100644
--- a/jarvis/ui/web/server.py
+++ b/jarvis/ui/web/server.py
@@ -139,6 +139,9 @@ class WebServer:
# the pending handle also makes repeated start() calls idempotent.
self._refresh_scheduler: Any | None = None
self._refresh_scheduler_start_handle: asyncio.Handle | None = None
+ # Local models self-check (badge only): started next to the refresh
+ # scheduler, i.e. after the serving path returned — never on boot.
+ self._local_models_health_monitor: Any | None = None
self._refresh_registry_tasks: set[asyncio.Task[Any]] = set()
self._refresh_scheduler_stopping = False
# Realtime transport pre-warm — scheduled at the end of start() so the
@@ -362,6 +365,7 @@ class WebServer:
from .friends_routes import router as friends_router
from .frontier_routes import router as frontier_router
from .grok_build_routes import router as grok_build_router
+ from .local_models_assistant_routes import router as local_models_assistant_router
from .local_models_routes import router as local_models_router
from .marketplace_publish_routes import router as marketplace_publish_router
from .marketplace_routes import router as marketplace_router
@@ -430,6 +434,8 @@ class WebServer:
# Local models section: inventory / unload / delete behind the
# pull-capable card (same capability gate as the pull routes).
app.include_router(local_models_router)
+ # Its setup assistant: run / session / test / benchmarks / health.
+ app.include_router(local_models_assistant_router)
app.include_router(antigravity_router)
app.include_router(claude_router)
app.include_router(grok_build_router)
@@ -2175,6 +2181,29 @@ class WebServer:
self._start_marketplace_refresh_scheduler
)
+ def _start_local_models_health_monitor(self) -> None:
+ """The Local models self-check task (best-effort, exactly once, off the boot path)."""
+ if self._local_models_health_monitor is not None:
+ return
+ try:
+ from jarvis.local_models.health_monitor import HealthMonitor
+
+ monitor = HealthMonitor(lambda: self.cfg)
+ monitor.start()
+ except Exception as exc: # noqa: BLE001 -- a badge must never block boot
+ logger.opt(exception=exc).warning("Local models health monitor did not start.")
+ return
+ self._local_models_health_monitor = monitor
+
+ async def _stop_local_models_health_monitor(self) -> None:
+ monitor = self._local_models_health_monitor
+ self._local_models_health_monitor = None
+ if monitor is not None:
+ try:
+ await monitor.stop()
+ except Exception as exc: # noqa: BLE001 -- shutdown stays best-effort
+ logger.opt(exception=exc).debug("Local models health monitor stop failed.")
+
def _start_marketplace_refresh_scheduler(self) -> None:
"""Create the periodic OAuth refresh task (best-effort, exactly once)."""
self._refresh_scheduler_start_handle = None
@@ -2681,6 +2710,7 @@ class WebServer:
# catalog/keyring reads or refresh network calls can begin. This shared
# hook covers normal desktop, headless, and direct WebServer starts.
self._schedule_marketplace_refresh_scheduler()
+ self._start_local_models_health_monitor()
# Same reasoning, and the same three boot paths: a headless or
# browser-only install has no desktop shell to warm the realtime
# transport for it.
@@ -3507,6 +3537,7 @@ class WebServer:
# Stop token refresh before the plugin registry so an in-flight refresh
# cannot enqueue a live-session rebuild while that registry is closing.
await self._stop_marketplace_refresh_scheduler()
+ await self._stop_local_models_health_monitor()
# Stop the skill watcher, otherwise the watchdog thread stays behind
# as a zombie and prevents the process from exiting.
diff --git a/tests/unit/local_models/test_health_monitor.py b/tests/unit/local_models/test_health_monitor.py
new file mode 100644
index 0000000000..e68e405502
--- /dev/null
+++ b/tests/unit/local_models/test_health_monitor.py
@@ -0,0 +1,165 @@
+"""The self-check: skip when there is nothing to watch, a bounded probe otherwise,
+a schedule of +10 min then every N hours — all with a fake clock and fake probes."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+from pathlib import Path
+from typing import Any
+
+import pytest
+
+from jarvis.core import config as cfg_mod
+from jarvis.core.config import BrainProviderConfig, JarvisConfig
+from jarvis.local_models import health_monitor as hm
+
+
[email protected](autouse=True)
+def _data_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
+ monkeypatch.setattr(cfg_mod, "DATA_DIR", tmp_path)
+ return tmp_path
+
+
+def _cfg(chat: str = "", hours: float | None = None) -> JarvisConfig:
+ cfg = JarvisConfig()
+ provider = BrainProviderConfig(model=chat, base_url="http://fake:11434")
+ if hours is not None:
+ provider.health_check_hours = hours
+ cfg.brain.providers["ollama"] = provider
+ return cfg
+
+
+def _probe(ok: bool):
+ async def _fn(_root: str) -> dict[str, Any]:
+ return {"ok": ok, "version": "0.32.15", "detail": "" if ok else "No Ollama answered."}
+
+ return _fn
+
+
+class _Result:
+ def __init__(self, status: str, detail: str = "") -> None:
+ self.status = status
+ self.detail = detail
+
+
+def _record(tmp_path: Path) -> dict[str, Any]:
+ return json.loads((tmp_path / "state" / "local_models_health.json").read_text("utf-8"))
+
+
+def test_interval_reads_the_one_config_field() -> None:
+ assert hm.interval_hours(_cfg()) == 6.0
+ assert hm.interval_hours(_cfg(hours=1.5)) == 1.5
+ assert hm.interval_hours(_cfg(hours=0)) == 6.0
+ assert hm.interval_hours(JarvisConfig()) == 6.0
+
+
+async def test_skips_when_the_server_is_down_and_nothing_is_configured(tmp_path: Path) -> None:
+ record = await hm.check_once(_cfg(), probe=_probe(False))
+ assert record is None
+ assert not (tmp_path / "state" / "local_models_health.json").exists()
+
+
+async def test_down_server_with_a_role_is_an_error(tmp_path: Path) -> None:
+ record = await hm.check_once(_cfg(chat="qwen3.5:4b"), probe=_probe(False))
+ assert record == {"status": "error", "reason": "No Ollama answered."}
+ assert _record(tmp_path)["status"] == "error"
+
+
+async def test_running_server_without_roles_needs_setup() -> None:
+ record = await hm.check_once(_cfg(), probe=_probe(True), persist=False)
+ assert record is not None and record["status"] == "needs_setup"
+
+
+async def test_one_generation_decides_ok_or_error(tmp_path: Path) -> None:
+ calls: list[str] = []
+
+ async def generate(_cfg: Any, model: str) -> Any:
+ calls.append(model)
+ return _Result("ok")
+
+ record = await hm.check_once(_cfg(chat="qwen3.5:4b"), probe=_probe(True), generate=generate)
+ assert record == {"status": "ok", "reason": ""} and calls == ["qwen3.5:4b"]
+ written = _record(tmp_path)
+ assert written["status"] == "ok" and written["last_ok"] == written["checked_at"]
+ assert written["since"] == written["checked_at"]
+
+ async def failing(_cfg: Any, model: str) -> Any:
+ return _Result("model_unavailable", "model not found")
+
+ record = await hm.check_once(_cfg(chat="qwen3.5:4b"), probe=_probe(True), generate=failing)
+ assert record == {"status": "error", "reason": "qwen3.5:4b: model not found"}
+ written = _record(tmp_path)
+ assert written["since"] == written["checked_at"], "the status changed, so 'since' restarts"
+ assert written["last_ok"] is not None, "the last good stamp survives"
+
+ async def hangs(_cfg: Any, model: str) -> Any:
+ await asyncio.sleep(3600)
+
+ hm_cap = hm.GENERATION_CAP_S
+ hm.GENERATION_CAP_S = 0.05
+ try:
+ record = await hm.check_once(
+ _cfg(chat="qwen3.5:4b"), probe=_probe(True), generate=hangs, persist=False
+ )
+ finally:
+ hm.GENERATION_CAP_S = hm_cap
+ assert record is not None and record["status"] == "error"
+ assert "did not answer" in record["reason"]
+
+
+def test_read_record_defaults_without_a_file() -> None:
+ assert hm.read_health_record() == {
+ "status": "unknown",
+ "reason": "",
+ "since": None,
+ "last_ok": None,
+ "checked_at": None,
+ }
+
+
+async def test_schedule_first_run_after_ten_minutes_then_every_n_hours() -> None:
+ sleeps: list[float] = []
+ checks: list[Any] = []
+ gate = asyncio.Event()
+
+ async def fake_sleep(seconds: float) -> None:
+ sleeps.append(seconds)
+ if len(sleeps) >= 3:
+ gate.set()
+ await asyncio.sleep(3600) # park until stop()
+ await asyncio.sleep(0)
+
+ async def fake_check(cfg: Any) -> None:
+ checks.append(cfg)
+
+ cfg = _cfg(hours=2.0)
+ monitor = hm.HealthMonitor(lambda: cfg, sleep=fake_sleep, check=fake_check)
+ monitor.start()
+ assert monitor.running
+ await asyncio.wait_for(gate.wait(), timeout=2.0)
+ await monitor.stop()
+ assert not monitor.running
+ assert sleeps[:3] == [hm.FIRST_RUN_DELAY_S, 7200.0, 7200.0]
+ assert monitor.runs == 2 and checks == [cfg, cfg]
+
+
+async def test_a_failing_check_never_ends_the_schedule() -> None:
+ sleeps: list[float] = []
+ gate = asyncio.Event()
+
+ async def fake_sleep(seconds: float) -> None:
+ sleeps.append(seconds)
+ if len(sleeps) >= 3:
+ gate.set()
+ await asyncio.sleep(3600)
+ await asyncio.sleep(0)
+
+ async def boom(_cfg: Any) -> None:
+ raise RuntimeError("probe exploded")
+
+ monitor = hm.HealthMonitor(lambda: _cfg(), sleep=fake_sleep, check=boom)
+ monitor.start()
+ await asyncio.wait_for(gate.wait(), timeout=2.0)
+ await monitor.stop()
+ assert monitor.runs == 2