Skip to content

Dodaios is an open-source agent OS for your desktop: it runs your coding agents, browser and apps by voice or chat, on your own machine. Formerly Personal Jarvis.

feat(brain): add a passive provider health record fed by real calls

rubenluetke10-beep220cc2fc69c5
diff --git a/jarvis/brain/manager.py b/jarvis/brain/manager.py
index 664ef796cd..45832e2a34 100644
--- a/jarvis/brain/manager.py
+++ b/jarvis/brain/manager.py
@@ -43,6 +43,9 @@ from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, Literal
from uuid import UUID, uuid4
+# Real turn outcomes feed the passive health record behind the status dots.
+from jarvis.brain import provider_health_ledger as _health_ledger
+
# ONE canonical billing/budget/quota marker list, shared with the test-badge
# classifier (provider_test.classify_provider_error) so the live fallback chain and
# the API-Keys badge can never disagree on "out of credits / over budget" (AP-22).
@@ -11943,6 +11946,11 @@ class BrainManager:
last_exc = exc
msg = str(exc)
kind = _classify_provider_error(msg, default="init_fail")
+ # The status dots read what real turns did; the record keeps
+ # only the classification, never this message (AP-34).
+ _health_ledger.record_failure(
+ prov_name, _health_ledger.MODALITY_BRAIN, exc, model=model
+ )
# On missing_key: remove provider from the chain for the rest
# of the session. Prevents each voice turn from running 8x
# sequentially against the same missing keys. Never for an
@@ -12315,6 +12323,9 @@ class BrainManager:
# tool was actually called this turn.
_turn_executed = set(executed)
used_provider, used_model = prov_name, model
+ _health_ledger.record_success(
+ prov_name, _health_ledger.MODALITY_BRAIN, model=model
+ )
# Bug C Fix (2026-04-29) — BrainTurnStarted/Completed publishen
# NUR wenn der Brain-Call erfolgreich war (Stream lieferte
@@ -12419,6 +12430,9 @@ class BrainManager:
# router bricked the whole turn even though OpenRouter was funded
# — AP-22). Only a genuinely transient 429 takes the cooldown path.
kind = _classify_provider_error(msg, default="call_fail")
+ _health_ledger.record_failure(
+ prov_name, _health_ledger.MODALITY_BRAIN, exc, model=model
+ )
if kind == "rate_limit":
self._rate_tracker.mark_rate_limited(prov_name, model)
log.warning("Rate-Limited %s(%s) — 30s Cooldown aktiviert", prov_name, model)
@@ -13559,11 +13573,17 @@ class BrainManager:
prompt, history=[], intent_level=intent, trace_id=trace_id,
turn_context=_scheduled_turn_context(turn_context, tools),
)
+ # This chain leads with the Tool Model, so its outcomes are the
+ # Tool Model tab's evidence.
+ _health_ledger.record_success(name, _health_ledger.MODALITY_TOOL, model=model)
return agg.text or ""
except Exception as exc:
kind = _classify_provider_error(str(exc), default="call_fail")
if kind not in _TASK_FALLBACK_KINDS:
raise
+ _health_ledger.record_failure(
+ name, _health_ledger.MODALITY_TOOL, exc, model=model
+ )
original = original or exc
failures.append(f"{name}: {_short_provider_error(exc)}")
remaining = attempts[index + 1:]
diff --git a/jarvis/brain/provider_health_ledger.py b/jarvis/brain/provider_health_ledger.py
new file mode 100644
index 0000000000..428f575a32
--- /dev/null
+++ b/jarvis/brain/provider_health_ledger.py
@@ -0,0 +1,613 @@
+"""Passive provider health — the last REAL outcome of every provider.
+
+Until 2026-09-30 the status dots on the API-Keys tabs, the sidebar, the dock
+and the chat composer's provider picker were fed by live probes: a real "hi"
+completion for the brain and the tool model, a real TTS synthesis, half a
+second of audio to a cloud recognizer, a real realtime session start. They ran
+on app start, on every window reload and after every key or provider event, so
+an app nobody was talking to still spent money on the user's keys (live log:
+one TTS synthesis plus one billed voice-session start per reload, about 17 a
+day, and a paid completion every time the chat opened).
+
+The dots now read THIS record instead. Every real call the app makes during
+normal use reports its outcome here — the brain turn, the tool-model step, the
+spoken sentence, the transcribed utterance, the realtime handshake — and the
+explicit, user-clicked "Test" button (plus the one check right after the user
+saves a key) reports its verdict too. Nothing in this module calls a provider.
+
+What is kept, per ``(provider id, modality)``:
+
+* the provider-test status of the outcome (``ok``, ``bad_key``, ``no_credits``,
+ ``rate_limited``, ``model_unavailable``, ``unreachable`` — the vocabulary of
+ :mod:`jarvis.brain.provider_test`, so the rollup maps it exactly as it mapped a
+ probe), the time it happened, the model when the caller knew it, and whether
+ it came from normal use or an explicit test.
+
+What is never kept: a provider's error body, a key, a prompt, a transcript
+(AP-34). A failure is classified the moment it is reported and only the
+classification survives.
+
+Rules the readers rely on:
+
+* An unrecognised failure (``error``) from normal use is NOT evidence about the
+ provider — it may be a tool, a cancelled turn or our own bug — so it is not
+ recorded. The explicit Test button's verdict is recorded whatever it says.
+* ``bad_key`` / ``no_credits`` are failures of the CREDENTIAL, which every
+ modality of the same provider id shares; a newer one on a sibling modality
+ wins over an older own outcome (see :meth:`ProviderHealthLedger.effective`).
+* Transient outcomes (``rate_limited``, ``unreachable``) age out after
+ :data:`TRANSIENT_TTL_S`, so a blip never paints a dot red for the rest of the
+ day while nobody uses that provider.
+* Saving a new key forgets what the old key did (:func:`forget_providers`), and
+ changing a model forgets the model-dependent outcomes of that modality.
+
+The record lives in memory and in ``DATA_DIR/state/provider_health.json`` so the
+dots survive a restart. The file is written only when an outcome CHANGES (or an
+unchanged one is older than :data:`_REFRESH_WRITE_S` on disk), never per call,
+so the hot voice path pays a dict update and nothing else (AP-9). Nothing
+initialises at import time (AP-26): the file is read on first use.
+"""
+from __future__ import annotations
+
+import json
+import logging
+import os
+import threading
+import time
+import uuid
+from collections.abc import Callable, Iterable
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from jarvis.brain import provider_test as _pt
+
+log = logging.getLogger(__name__)
+
+LEDGER_FILE_NAME = "provider_health.json"
+_FILE_VERSION = 1
+
+# ── Modalities ───────────────────────────────────────────────────────────────
+MODALITY_BRAIN = "brain"
+MODALITY_TOOL = "tool"
+MODALITY_TTS = "tts"
+MODALITY_STT = "stt"
+MODALITY_REALTIME = "realtime"
+MODALITIES: tuple[str, ...] = (
+ MODALITY_BRAIN,
+ MODALITY_TOOL,
+ MODALITY_TTS,
+ MODALITY_STT,
+ MODALITY_REALTIME,
+)
+
+# ── Sources ──────────────────────────────────────────────────────────────────
+SOURCE_USE = "use" # a call the app made during normal use
+SOURCE_TEST = "test" # the user clicked Test / saved a key / picked a model
+SOURCES: tuple[str, ...] = (SOURCE_USE, SOURCE_TEST)
+
+# ── Which outcomes are evidence ──────────────────────────────────────────────
+#: Outcomes a normal call may record. ``error`` is excluded (not evidence about
+#: the provider) and so is ``not_configured`` (the credential-presence check
+#: already answers that, and a missing key is not a provider state).
+USE_STATUSES: frozenset[str] = frozenset(
+ {
+ _pt.OK,
+ _pt.BAD_KEY,
+ _pt.NO_CREDITS,
+ _pt.RATE_LIMITED,
+ _pt.MODEL_UNAVAILABLE,
+ _pt.UNREACHABLE,
+ }
+)
+#: An explicit test is a deliberate, fresh verdict: its ``error`` counts too.
+TEST_STATUSES: frozenset[str] = USE_STATUSES | {_pt.ERROR}
+#: Failures of the credential itself — shared by every modality of a provider.
+CREDENTIAL_STATUSES: frozenset[str] = frozenset({_pt.BAD_KEY, _pt.NO_CREDITS})
+#: Failures that clear on their own; they expire instead of lingering.
+TRANSIENT_STATUSES: frozenset[str] = frozenset({_pt.RATE_LIMITED, _pt.UNREACHABLE})
+#: How long a transient failure keeps its dot.
+TRANSIENT_TTL_S = 30 * 60.0
+#: An unchanged outcome is re-written to disk at most this often, so its age
+#: survives a restart roughly right without a write per call.
+_REFRESH_WRITE_S = 15 * 60.0
+
+# Plugin names that differ from the provider-card id they belong to. The
+# OpenRouter speech plugin calls itself ``openrouter`` — the same id as the
+# OpenRouter BRAIN card — while its card is ``openrouter-tts``.
+_ALIASES: dict[tuple[str, str], str] = {
+ (MODALITY_TTS, "openrouter"): "openrouter-tts",
+ (MODALITY_TTS, "openrouter_tts"): "openrouter-tts",
+ (MODALITY_TTS, "open-router-tts"): "openrouter-tts",
+}
+
+
+def _norm_provider(provider: str | None, modality: str) -> str:
+ key = (provider or "").strip().casefold()
+ return _ALIASES.get((modality, key), key)
+
+
+@dataclass(frozen=True, slots=True)
+class Outcome:
+ """One remembered outcome. Never carries an error body or a secret."""
+
+ provider: str
+ modality: str
+ status: str
+ at: float
+ source: str = SOURCE_USE
+ model: str = ""
+
+ def to_json(self) -> dict[str, Any]:
+ return {
+ "provider": self.provider,
+ "modality": self.modality,
+ "status": self.status,
+ "at": self.at,
+ "source": self.source,
+ "model": self.model,
+ }
+
+ @classmethod
+ def from_json(cls, raw: Any) -> Outcome | None:
+ """Parse one persisted entry; anything malformed is dropped."""
+ if not isinstance(raw, dict):
+ return None
+ provider = str(raw.get("provider") or "").strip()
+ modality = str(raw.get("modality") or "")
+ status = str(raw.get("status") or "")
+ source = str(raw.get("source") or SOURCE_USE)
+ try:
+ at = float(raw.get("at"))
+ except (TypeError, ValueError):
+ return None
+ if not provider or modality not in MODALITIES or source not in SOURCES:
+ return None
+ allowed = TEST_STATUSES if source == SOURCE_TEST else USE_STATUSES
+ if status not in allowed:
+ return None
+ return cls(
+ provider=provider,
+ modality=modality,
+ status=status,
+ at=at,
+ source=source,
+ model=str(raw.get("model") or ""),
+ )
+
+
+def failure_status(error: BaseException | str | None) -> str:
+ """Classify a failure into the provider-test vocabulary.
+
+ The one classifier the Test button uses, so a real call and a test can
+ never disagree about what "401" or "insufficient_quota" means.
+ """
+ if error is None:
+ return _pt.ERROR
+ if isinstance(error, BaseException):
+ text = f"{type(error).__name__}: {error}"
+ else:
+ text = str(error)
+ return _pt.classify_provider_error(text)
+
+
+class ProviderHealthLedger:
+ """Thread-safe record of the last real outcome per provider and modality."""
+
+ def __init__(
+ self,
+ path: Path | None = None,
+ *,
+ clock: Callable[[], float] = time.time,
+ ) -> None:
+ self._path = path
+ self._clock = clock
+ self._lock = threading.Lock()
+ self._write_lock = threading.Lock()
+ self._entries: dict[tuple[str, str], Outcome] = {}
+ self._written_at: dict[tuple[str, str], float] = {}
+ self._loaded = path is None
+ self._version = 0
+
+ # ── reading ──────────────────────────────────────────────────────────
+ @property
+ def version(self) -> int:
+ """Bumps whenever a remembered status appears, changes or is forgotten."""
+ with self._lock:
+ self._ensure_loaded_locked()
+ return self._version
+
+ def get(self, provider: str, modality: str) -> Outcome | None:
+ with self._lock:
+ self._ensure_loaded_locked()
+ return self._entries.get((_norm_provider(provider, modality), modality))
+
+ def effective(
+ self,
+ provider: str,
+ modality: str,
+ *,
+ now: float | None = None,
+ ) -> Outcome | None:
+ """The outcome that should colour ``provider``'s ``modality`` right now.
+
+ * a transient failure older than :data:`TRANSIENT_TTL_S` is ignored;
+ * the key is shared by every modality of the same provider id, so a
+ credential failure recorded on ANOTHER modality wins when it is newer
+ than this modality's own outcome — and this modality's own credential
+ failure is void once a sibling answered after it;
+ * otherwise this modality's own outcome, or ``None`` (never used yet).
+ """
+ current = self._clock() if now is None else now
+ key_provider = _norm_provider(provider, modality)
+ with self._lock:
+ self._ensure_loaded_locked()
+ own = self._entries.get((key_provider, modality))
+ siblings = [
+ outcome
+ for (p, m), outcome in self._entries.items()
+ if p == key_provider and m != modality
+ ]
+ if own is not None and self._expired(own, current):
+ own = None
+ newest_credential: Outcome | None = None
+ newest_ok: Outcome | None = None
+ for sibling in siblings:
+ if sibling.status in CREDENTIAL_STATUSES:
+ if newest_credential is None or sibling.at > newest_credential.at:
+ newest_credential = sibling
+ elif sibling.status == _pt.OK:
+ if newest_ok is None or sibling.at > newest_ok.at:
+ newest_ok = sibling
+ if newest_credential is not None and (own is None or newest_credential.at > own.at):
+ if newest_ok is None or newest_credential.at > newest_ok.at:
+ return newest_credential
+ if (
+ own is not None
+ and own.status in CREDENTIAL_STATUSES
+ and newest_ok is not None
+ and newest_ok.at > own.at
+ ):
+ return None
+ return own
+
+ def snapshot(self) -> list[Outcome]:
+ with self._lock:
+ self._ensure_loaded_locked()
+ return list(self._entries.values())
+
+ # ── writing ──────────────────────────────────────────────────────────
+ def record(
+ self,
+ provider: str,
+ modality: str,
+ status: str,
+ *,
+ source: str = SOURCE_USE,
+ model: str | None = None,
+ ) -> bool:
+ """Remember one outcome. Returns whether it was recorded.
+
+ Never raises: a health record must not cost the call it describes.
+ """
+ if modality not in MODALITIES or source not in SOURCES:
+ return False
+ allowed = TEST_STATUSES if source == SOURCE_TEST else USE_STATUSES
+ if status not in allowed:
+ return False
+ key_provider = _norm_provider(provider, modality)
+ if not key_provider:
+ return False
+ now = self._clock()
+ outcome = Outcome(
+ provider=key_provider,
+ modality=modality,
+ status=status,
+ at=now,
+ source=source,
+ model=(model or "").strip(),
+ )
+ key = (key_provider, modality)
+ with self._lock:
+ self._ensure_loaded_locked()
+ previous = self._entries.get(key)
+ self._entries[key] = outcome
+ changed = previous is None or previous.status != status
+ if changed:
+ self._version += 1
+ due = changed or now - self._written_at.get(key, 0.0) >= _REFRESH_WRITE_S
+ if due:
+ self._written_at[key] = now
+ payload = self._payload_locked() if due else None
+ if changed:
+ log.info(
+ "Provider health: %s/%s %s -> %s (%s)",
+ key_provider,
+ modality,
+ previous.status if previous is not None else "none",
+ status,
+ source,
+ )
+ if payload is not None:
+ self._persist(payload)
+ return True
+
+ def record_failure(
+ self,
+ provider: str,
+ modality: str,
+ error: BaseException | str | None,
+ *,
+ source: str = SOURCE_USE,
+ model: str | None = None,
+ ) -> bool:
+ """Classify ``error`` and remember it when it is evidence (see module doc)."""
+ return self.record(
+ provider, modality, failure_status(error), source=source, model=model
+ )
+
+ def forget(
+ self,
+ providers: Iterable[str],
+ *,
+ modality: str | None = None,
+ keep_credential_failures: bool = False,
+ ) -> int:
+ """Drop outcomes of ``providers`` (optionally one ``modality``).
+
+ ``keep_credential_failures`` keeps ``bad_key`` / ``no_credits``: a model
+ change says nothing about whether the key works.
+ """
+ wanted: set[str] = set()
+ for provider in providers:
+ for m in MODALITIES:
+ normalized = _norm_provider(provider, m)
+ if normalized:
+ wanted.add(normalized)
+ if not wanted:
+ return 0
+ with self._lock:
+ self._ensure_loaded_locked()
+ doomed = [
+ key
+ for key, outcome in self._entries.items()
+ if key[0] in wanted
+ and (modality is None or key[1] == modality)
+ and not (keep_credential_failures and outcome.status in CREDENTIAL_STATUSES)
+ ]
+ for key in doomed:
+ self._entries.pop(key, None)
+ self._written_at.pop(key, None)
+ if doomed:
+ self._version += 1
+ payload = self._payload_locked() if doomed else None
+ if payload is not None:
+ self._persist(payload)
+ return len(doomed)
+
+ # ── internals ────────────────────────────────────────────────────────
+ @staticmethod
+ def _expired(outcome: Outcome, now: float) -> bool:
+ return outcome.status in TRANSIENT_STATUSES and now - outcome.at > TRANSIENT_TTL_S
+
+ def _payload_locked(self) -> dict[str, Any]:
+ return {
+ "version": _FILE_VERSION,
+ "outcomes": [outcome.to_json() for outcome in self._entries.values()],
+ }
+
+ def _ensure_loaded_locked(self) -> None:
+ if self._loaded:
+ return
+ self._loaded = True
+ path = self._path
+ if path is None:
+ return
+ try:
+ raw = json.loads(path.read_text(encoding="utf-8"))
+ except FileNotFoundError:
+ return
+ except (OSError, ValueError):
+ # A damaged record only costs the history; the next real call
+ # writes a fresh one.
+ log.info("Provider health: %s unreadable, starting empty", path, exc_info=True)
+ return
+ items = raw.get("outcomes") if isinstance(raw, dict) else None
+ if not isinstance(items, list):
+ return
+ for item in items:
+ outcome = Outcome.from_json(item)
+ if outcome is None:
+ continue
+ key = (outcome.provider, outcome.modality)
+ self._entries[key] = outcome
+ self._written_at[key] = outcome.at
+ if self._entries:
+ self._version += 1
+
+ def _persist(self, payload: dict[str, Any]) -> None:
+ path = self._path
+ if path is None:
+ return
+ with self._write_lock:
+ temporary = path.with_name(f".{path.name}.{os.getpid()}.{uuid.uuid4().hex}.tmp")
+ try:
+ path.parent.mkdir(parents=True, exist_ok=True)
+ temporary.write_text(json.dumps(payload, indent=2), encoding="utf-8")
+ os.replace(temporary, path)
+ except OSError:
+ log.warning("Provider health: %s not written", path, exc_info=True)
+ finally:
+ try:
+ temporary.unlink(missing_ok=True)
+ except OSError:
+ log.debug("Provider health: temporary file cleanup failed", exc_info=True)
+
+
+# ── process-wide instance ────────────────────────────────────────────────────
+_ledger: ProviderHealthLedger | None = None
+_ledger_guard = threading.Lock()
+
+
+def ledger_path() -> Path:
+ """``DATA_DIR/state/provider_health.json`` — resolved at call time."""
+ from jarvis.core import config as cfg_mod # lazy: tests redirect DATA_DIR
+
+ return Path(cfg_mod.DATA_DIR) / "state" / LEDGER_FILE_NAME
+
+
+def get_ledger() -> ProviderHealthLedger:
+ """The process-wide ledger, created on first use (never at import)."""
+ global _ledger
+ with _ledger_guard:
+ if _ledger is None:
+ _ledger = ProviderHealthLedger(ledger_path())
+ return _ledger
+
+
+def set_ledger(ledger: ProviderHealthLedger | None) -> None:
+ """Swap the process-wide ledger (tests; ``None`` rebuilds on next use)."""
+ global _ledger
+ with _ledger_guard:
+ _ledger = ledger
+
+
+def _safely(action: Callable[[], Any], what: str) -> Any:
+ """Run a ledger action for a caller on a live call path.
+
+ A health record must never cost the call it describes, so a failure here
+ is logged (once per occurrence, at warning) and swallowed on purpose.
+ """
+ try:
+ return action()
+ except Exception: # noqa: BLE001 — see docstring: the call path must not pay
+ log.warning("Provider health: %s failed", what, exc_info=True)
+ return None
+
+
+def record_success(
+ provider: str | None,
+ modality: str,
+ *,
+ source: str = SOURCE_USE,
+ model: str | None = None,
+) -> None:
+ """A real call to ``provider`` answered."""
+ if not provider:
+ return
+ _safely(
+ lambda: get_ledger().record(provider, modality, _pt.OK, source=source, model=model),
+ "record_success",
+ )
+
+
+def record_failure(
+ provider: str | None,
+ modality: str,
+ error: BaseException | str | None,
+ *,
+ source: str = SOURCE_USE,
+ model: str | None = None,
+) -> None:
+ """A real call to ``provider`` failed with ``error`` (classified, never stored)."""
+ if not provider:
+ return
+ _safely(
+ lambda: get_ledger().record_failure(
+ provider, modality, error, source=source, model=model
+ ),
+ "record_failure",
+ )
+
+
+def record_status(
+ provider: str | None,
+ modality: str,
+ status: str,
+ *,
+ source: str = SOURCE_USE,
+ model: str | None = None,
+) -> None:
+ """Remember an already-classified outcome (a Test button verdict)."""
+ if not provider:
+ return
+ _safely(
+ lambda: get_ledger().record(provider, modality, status, source=source, model=model),
+ "record_status",
+ )
+
+
+def forget_providers(
+ providers: Iterable[str],
+ *,
+ modality: str | None = None,
+ keep_credential_failures: bool = False,
+) -> None:
+ """Forget what ``providers`` did — a new key or model starts clean."""
+ names = [p for p in providers if p]
+ if not names:
+ return
+ _safely(
+ lambda: get_ledger().forget(
+ names, modality=modality, keep_credential_failures=keep_credential_failures
+ ),
+ "forget_providers",
+ )
+
+
+def effective_outcome(provider: str | None, modality: str) -> Outcome | None:
+ """See :meth:`ProviderHealthLedger.effective`; ``None`` on any failure."""
+ if not provider:
+ return None
+ return _safely(lambda: get_ledger().effective(provider, modality), "effective_outcome")
+
+
+def ledger_version() -> int:
+ """Current ledger version for cache keys; ``0`` if the ledger is unreadable."""
+ version = _safely(lambda: get_ledger().version, "ledger_version")
+ return int(version or 0)
+
+
+def modality_for_tier(tier: str | None) -> str | None:
+ """The modality a provider-card tier's calls are recorded under.
+
+ ``None`` for a tier this ledger does not track (the dictation wording pass
+ is judged by credential presence alone).
+ """
+ return {
+ "brain": MODALITY_BRAIN,
+ "tts": MODALITY_TTS,
+ "stt": MODALITY_STT,
+ "realtime": MODALITY_REALTIME,
+ }.get((tier or "").strip().lower())
+
+
+__all__ = [
+ "CREDENTIAL_STATUSES",
+ "LEDGER_FILE_NAME",
+ "MODALITIES",
+ "MODALITY_BRAIN",
+ "MODALITY_REALTIME",
+ "MODALITY_STT",
+ "MODALITY_TOOL",
+ "MODALITY_TTS",
+ "Outcome",
+ "ProviderHealthLedger",
+ "SOURCE_TEST",
+ "SOURCE_USE",
+ "TEST_STATUSES",
+ "TRANSIENT_STATUSES",
+ "TRANSIENT_TTL_S",
+ "USE_STATUSES",
+ "effective_outcome",
+ "failure_status",
+ "forget_providers",
+ "get_ledger",
+ "ledger_path",
+ "ledger_version",
+ "modality_for_tier",
+ "record_failure",
+ "record_status",
+ "record_success",
+ "set_ledger",
+]
diff --git a/jarvis/cu/brain_call.py b/jarvis/cu/brain_call.py
index eece6dd5be..f374ae9bc2 100644
--- a/jarvis/cu/brain_call.py
+++ b/jarvis/cu/brain_call.py
@@ -430,6 +430,7 @@ async def call_vision_brain(
provider, model, idx,
)
_log_serving(provider, model)
+ selector.record_success(provider, model)
return reply
# Stale-dead-flag resilience: retry every REGISTERED vision provider once,
@@ -463,6 +464,7 @@ async def call_vision_brain(
provider, model,
)
_log_serving(provider, model)
+ selector.record_success(provider, model)
return reply
raise CUNoVisionProviderError(
diff --git a/jarvis/harness/computer_use_planner.py b/jarvis/harness/computer_use_planner.py
index b1dac9f219..6e4e1040c0 100644
--- a/jarvis/harness/computer_use_planner.py
+++ b/jarvis/harness/computer_use_planner.py
@@ -152,12 +152,22 @@ class ComputerUsePlannerSelector:
def record_empty(self, provider: str, model: str | None) -> None:
self._record(provider, model, "empty", "empty response")
+ def record_success(self, provider: str, model: str | None) -> None:
+ """A planner step answered: the Tool Model tab's status dot reads it."""
+ from jarvis.brain import provider_health_ledger as ledger # noqa: PLC0415
+
+ ledger.record_success(provider, ledger.MODALITY_TOOL, model=model)
+
def record_failure(
self, provider: str, model: str | None, exc: Exception,
) -> None:
+ from jarvis.brain import provider_health_ledger as ledger # noqa: PLC0415
+
detail = str(exc)
kind = self._classify_failure(exc, detail)
self._record(provider, model, kind, detail[:200])
+ # Only the classification is kept there, never ``detail`` (AP-34).
+ ledger.record_failure(provider, ledger.MODALITY_TOOL, exc, model=model)
if kind == "rate_limit":
rate_tracker = getattr(self.manager, "_rate_tracker", None)
diff --git a/jarvis/harness/screenshot_only_loop.py b/jarvis/harness/screenshot_only_loop.py
index 354f263ed5..a56efc935d 100644
--- a/jarvis/harness/screenshot_only_loop.py
+++ b/jarvis/harness/screenshot_only_loop.py
@@ -1627,6 +1627,7 @@ async def _call_brain(
"ComputerUseLoop fallback hit: %s(%s) after %d skipped provider(s)",
provider, model, idx,
)
+ selector.record_success(provider, model)
return text
except Exception as exc: # noqa: BLE001
selector.record_failure(provider, model, exc)
@@ -1666,6 +1667,7 @@ async def _call_brain(
"— normal chain had no vision provider (stale dead-flag?)",
provider, model,
)
+ selector.record_success(provider, model)
return text
except Exception as exc: # noqa: BLE001
selector.record_failure(provider, model, exc)
diff --git a/jarvis/harness/screenshot_only_loop_stable.py b/jarvis/harness/screenshot_only_loop_stable.py
index 0949c34734..1988892651 100644
--- a/jarvis/harness/screenshot_only_loop_stable.py
+++ b/jarvis/harness/screenshot_only_loop_stable.py
@@ -1412,6 +1412,7 @@ async def _call_brain(
"ComputerUseLoop fallback hit: %s(%s) after %d skipped provider(s)",
provider, model, idx,
)
+ selector.record_success(provider, model)
return text
except Exception as exc: # noqa: BLE001
selector.record_failure(provider, model, exc)
@@ -1451,6 +1452,7 @@ async def _call_brain(
"— normal chain had no vision provider (stale dead-flag?)",
provider, model,
)
+ selector.record_success(provider, model)
return text
except Exception as exc: # noqa: BLE001
selector.record_failure(provider, model, exc)
diff --git a/jarvis/live/session.py b/jarvis/live/session.py
index 0c789e0ee6..09e878f38c 100644
--- a/jarvis/live/session.py
+++ b/jarvis/live/session.py
@@ -447,6 +447,7 @@ class LiveVoiceSession:
"Check the selected model and account access."
)
self._pump_task = asyncio.create_task(self._pump(), name="live-events")
+ from jarvis.brain import provider_health_ledger as health_ledger
from jarvis.core.events import (
RealtimeSessionReady,
VoiceSessionStarted,
@@ -455,6 +456,11 @@ class LiveVoiceSession:
from jarvis.live.runtime import register
register(self)
+ # The started session is the Realtime tab's evidence; its status
+ # dot never opens a (billed) session of its own.
+ health_ledger.record_success(
+ self.active_provider, health_ledger.MODALITY_REALTIME, model=profile.model
+ )
if self._bus is not None:
if not self._parent_owned:
await self._bus.publish(
@@ -509,6 +515,7 @@ class LiveVoiceSession:
Live 2026-09-29: an empty API balance made every wake end after a
second with no word, which read as a broken wake word.
"""
+ from jarvis.brain import provider_health_ledger as health_ledger
from jarvis.brain.provider_test import (
NO_CREDITS,
RATE_LIMITED,
@@ -517,6 +524,9 @@ class LiveVoiceSession:
from jarvis.realtime.session import _handshake_failure_message
status = classify_provider_error(str(exc))
+ health_ledger.record_status(
+ self.active_provider, health_ledger.MODALITY_REALTIME, status
+ )
cause = {NO_CREDITS: "no_credits", RATE_LIMITED: "rate_limited"}.get(
status, "unavailable"
)
diff --git a/jarvis/plugins/tts/cartesia_tts.py b/jarvis/plugins/tts/cartesia_tts.py
index 04430e7130..c1f45cff2a 100644
--- a/jarvis/plugins/tts/cartesia_tts.py
+++ b/jarvis/plugins/tts/cartesia_tts.py
@@ -186,6 +186,9 @@ class CartesiaTTS:
voice_id = self._resolve_voice(text, voice, language_code)
self.last_voice = voice_id
self.last_voice_provider = self.name
+ # What this provider failed with when a fallback voice spoke instead
+ # (read by the speech meter for the provider-health dots).
+ self.last_failure: str | None = None
log = logging.getLogger("jarvis.tts.cartesia")
@@ -219,6 +222,7 @@ class CartesiaTTS:
try:
pcm = await task
except _CartesiaFatalError as exc:
+ self.last_failure = f"{type(exc).__name__}: {exc}"
self._quota_blocked_until = time.monotonic() + _QUOTA_COOLDOWN_S
log.warning(
"Cartesia quota/auth error (%s) -- fallback for %.0f min.",
diff --git a/jarvis/plugins/tts/elevenlabs_tts.py b/jarvis/plugins/tts/elevenlabs_tts.py
index db25bc2a30..19a389dff5 100644
--- a/jarvis/plugins/tts/elevenlabs_tts.py
+++ b/jarvis/plugins/tts/elevenlabs_tts.py
@@ -176,6 +176,9 @@ class ElevenLabsTTS:
log = logging.getLogger("jarvis.tts.elevenlabs")
self.last_voice = voice
self.last_voice_provider = self.name
+ # What this provider failed with when a fallback voice spoke instead
+ # (read by the speech meter for the provider-health dots).
+ self.last_failure: str | None = None
# Cooldown active? Gemini first, then SAPI5 — never stay silent.
if self._quota_blocked_until and time.monotonic() < self._quota_blocked_until:
@@ -201,6 +204,7 @@ class ElevenLabsTTS:
yield chunk
except Exception as exc: # noqa: BLE001
msg = str(exc)
+ self.last_failure = f"{type(exc).__name__}: {msg}"
low = msg.lower()
if (
"quota" in low
diff --git a/jarvis/plugins/tts/fallback_tts.py b/jarvis/plugins/tts/fallback_tts.py
index 5df26a5e1e..7919b155f8 100644
--- a/jarvis/plugins/tts/fallback_tts.py
+++ b/jarvis/plugins/tts/fallback_tts.py
@@ -71,6 +71,19 @@ class FallbackTTS:
speaker, "name", None
)
+ _last_failure: str | None = None
+
+ @property
+ def last_failure(self) -> str | None:
+ """What the asked provider failed with when another voice spoke the
+ last utterance, else ``None``. In memory only: the speech meter hands
+ it to the provider-health record, which keeps just a classification.
+ A primary that fell back internally reports its own ``last_failure``.
+ """
+ if self._last_speaker is self._fallback:
+ return self._last_failure
+ return getattr(self._primary, "last_failure", None)
+
@property
def primary(self) -> Any:
return self._primary
@@ -124,6 +137,7 @@ class FallbackTTS:
produced = 0
self._last_speaker = self._primary
+ self._last_failure = None
try:
async for chunk in self._primary.synthesize(
text, voice=voice, language_code=language_code
@@ -131,6 +145,7 @@ class FallbackTTS:
produced += 1
yield chunk
except Exception as exc: # noqa: BLE001 — any primary failure -> fallback
+ self._last_failure = f"{type(exc).__name__}: {exc}"
if produced:
# Audio already reached the speaker; restarting from the
# fallback would replay the opening words. Surface the error
diff --git a/jarvis/plugins/tts/grok_voice_tts.py b/jarvis/plugins/tts/grok_voice_tts.py
index 828ff1e686..3278ca1b82 100644
--- a/jarvis/plugins/tts/grok_voice_tts.py
+++ b/jarvis/plugins/tts/grok_voice_tts.py
@@ -201,6 +201,9 @@ class GrokVoiceTTS:
log = logging.getLogger("jarvis.tts.grok-voice")
self.last_voice = voice
self.last_voice_provider = self.name
+ # What this provider failed with when a fallback voice spoke instead
+ # (read by the speech meter for the provider-health dots).
+ self.last_failure: str | None = None
# Cooldown active? First Gemini, then SAPI5 — never stay silent.
if self._quota_blocked_until and time.monotonic() < self._quota_blocked_until:
@@ -234,6 +237,7 @@ class GrokVoiceTTS:
except _GrokFatalError as exc:
# 401/403/429 → arm the cooldown, cancel remaining tasks,
# fall back for the rest of the text.
+ self.last_failure = f"{type(exc).__name__}: {exc}"
self._quota_blocked_until = time.monotonic() + _QUOTA_COOLDOWN_S
log.warning(
"Grok Voice quota/auth error (%s) — falling back for %.0f min.",
diff --git a/jarvis/plugins/tts/inworld_tts.py b/jarvis/plugins/tts/inworld_tts.py
index c2b4c2978e..ec4c016c0e 100644
--- a/jarvis/plugins/tts/inworld_tts.py
+++ b/jarvis/plugins/tts/inworld_tts.py
@@ -201,6 +201,9 @@ class InworldTTS:
voice_id = self._resolve_voice(text, voice, language_code)
self.last_voice = voice_id
self.last_voice_provider = self.name
+ # What this provider failed with when a fallback voice spoke instead
+ # (read by the speech meter for the provider-health dots).
+ self.last_failure: str | None = None
log = logging.getLogger("jarvis.tts.inworld")
@@ -230,6 +233,7 @@ class InworldTTS:
try:
pcm = await task
except _InworldFatalError as exc:
+ self.last_failure = f"{type(exc).__name__}: {exc}"
self._quota_blocked_until = time.monotonic() + _QUOTA_COOLDOWN_S
log.warning(
"Inworld quota/auth error (%s) -- fallback for %.0f min.",
diff --git a/jarvis/realtime/session.py b/jarvis/realtime/session.py
index edc5759232..d8a0887c3a 100644
--- a/jarvis/realtime/session.py
+++ b/jarvis/realtime/session.py
@@ -3784,6 +3784,13 @@ class RealtimeVoiceSession:
a terminal event through the same session cannot make it healthy.
"""
status = classify_provider_error(message)
+ # Every realtime provider failure passes here: record its class (never
+ # the message, AP-34) for the Realtime tab's status dot.
+ from jarvis.brain import provider_health_ledger as health_ledger
+
+ health_ledger.record_status(
+ self._provider_id(provider), health_ledger.MODALITY_REALTIME, status
+ )
if status in _CREDENTIAL_TERMINAL_STATUSES:
self._blocked_credential_families.add(
self._credential_family(provider)
@@ -7935,6 +7942,15 @@ class RealtimeVoiceSession:
pass
async def _publish_ready(self) -> None:
+ # The accepted handshake is the Realtime tab's evidence: its status dot
+ # reads this instead of opening a (billed) session of its own.
+ from jarvis.brain import provider_health_ledger as health_ledger
+
+ health_ledger.record_success(
+ self.active_provider,
+ health_ledger.MODALITY_REALTIME,
+ model=self._active_model,
+ )
if self._bus is None:
return
try:
diff --git a/jarvis/speech/stt_fallback.py b/jarvis/speech/stt_fallback.py
index e1f9390088..ac694a85ee 100644
--- a/jarvis/speech/stt_fallback.py
+++ b/jarvis/speech/stt_fallback.py
@@ -168,6 +168,14 @@ class FallbackSTT:
return ""
_last_used = ""
+ _last_failure: str | None = None
+
+ @property
+ def last_failure(self) -> str | None:
+ """What the primary failed with when a fallback produced the latest
+ transcript, else ``None``. In memory only; the speech meter hands it to
+ the provider-health record, which keeps just its classification."""
+ return self._last_failure
# ------------------------------------------------------------------
# Cooldown bookkeeping
@@ -216,6 +224,7 @@ class FallbackSTT:
# ------------------------------------------------------------------
async def _call(self, method: str, *args: Any, **kwargs: Any) -> Any:
first_error: BaseException | None = None
+ self._last_failure = None
for name, preloaded in self._order():
try:
provider = preloaded if preloaded is not None else self._instance(name)
@@ -241,6 +250,8 @@ class FallbackSTT:
raise
if first_error is None:
first_error = exc
+ if name == self._primary_name and self._last_failure is None:
+ self._last_failure = f"{type(exc).__name__}: {exc}"
self._penalize(name, kind)
log.warning(
"STT provider %s failed (%s: %s) — crossing to the next "
diff --git a/jarvis/speech/usage_meter.py b/jarvis/speech/usage_meter.py
index c956d74b3c..f91dd0c483 100644
--- a/jarvis/speech/usage_meter.py
+++ b/jarvis/speech/usage_meter.py
@@ -229,6 +229,43 @@ class _MeteredProvider:
else:
log.debug("Speech usage sink failed again (%s).", exc)
+ def _report_health(
+ self,
+ *,
+ failure: BaseException | None,
+ answered: bool,
+ ) -> None:
+ """Feed the passive provider-health record behind the status dots.
+
+ The dots never probe a provider (a probe bills the key); they read what
+ real calls did, and this wrapper sees every real speech call. A failure
+ is recorded against the provider that was asked, a success against the
+ vendor that actually spoke; when a fallback spoke, the failure the
+ asked provider absorbed (``last_failure``, when it exposes one) is
+ recorded too. Only the classification is kept (AP-34).
+
+ Never raises: like the usage report, health bookkeeping must not cost
+ an utterance, so a failure here is logged and dropped on purpose.
+ """
+ try:
+ from jarvis.brain import provider_health_ledger as ledger
+
+ modality = ledger.MODALITY_TTS if self._stage == STAGE_TTS else ledger.MODALITY_STT
+ asked = self.name
+ if failure is not None:
+ ledger.record_failure(asked, modality, failure)
+ return
+ if not answered:
+ return
+ spoke = self._provider_label() or asked
+ if asked and spoke.casefold() != asked.casefold():
+ absorbed = getattr(self._inner, "last_failure", None)
+ if absorbed:
+ ledger.record_failure(asked, modality, str(absorbed))
+ ledger.record_success(spoke, modality)
+ except Exception as exc: # noqa: BLE001 — see docstring: speech must not pay
+ log.debug("Speech health record failed (%s).", exc)
+
def _provider_label(self) -> str:
"""Who actually did the work.
@@ -279,10 +316,14 @@ class MeteredTTS(_MeteredProvider):
kwargs["language_code"] = language_code
audio_ms = 0.0
+ failure: Exception | None = None
try:
async for chunk in self._inner.synthesize(text, **kwargs):
audio_ms += _chunk_ms(chunk)
yield chunk
+ except Exception as exc:
+ failure = exc
+ raise
finally:
# Reached on exhaustion, on an exception, on cancellation, and on
# the ``aclose()`` of a generator the caller walked away from. The
@@ -292,6 +333,9 @@ class MeteredTTS(_MeteredProvider):
chars=chars,
audio_ms=audio_ms,
)
+ # Audio arrived = the provider answered, even if the caller then
+ # walked away; a cancellation with no audio is no evidence at all.
+ self._report_health(failure=failure, answered=audio_ms > 0)
def _tts_rate_key(self, voice: str | None) -> str:
"""The identifier the TTS rate actually depends on.
@@ -317,14 +361,22 @@ class MeteredSTT(_MeteredProvider):
async def transcribe(self, audio: AsyncIterator[Any], *args: Any, **kwargs: Any) -> Any:
counter = _CountingAudio(audio)
+ failure: Exception | None = None
+ answered = False
try:
- return await self._inner.transcribe(counter, *args, **kwargs)
+ result = await self._inner.transcribe(counter, *args, **kwargs)
+ answered = True
+ return result
+ except Exception as exc:
+ failure = exc
+ raise
finally:
self._report(
model_or_voice=self._stt_rate_key(),
chars=0,
audio_ms=counter.audio_ms,
)
+ self._report_health(failure=failure, answered=answered)
# An async generator for the same reason ``synthesize`` is one: the
# implementations are generators, callers iterate the return value without
@@ -333,15 +385,22 @@ class MeteredSTT(_MeteredProvider):
self, audio: AsyncIterator[Any], *args: Any, **kwargs: Any
) -> AsyncIterator[Any]:
counter = _CountingAudio(audio)
+ failure: Exception | None = None
+ answered = False
try:
async for transcript in self._inner.stream_transcribe(counter, *args, **kwargs):
+ answered = True
yield transcript
+ except Exception as exc:
+ failure = exc
+ raise
finally:
self._report(
model_or_voice=self._stt_rate_key(),
chars=0,
audio_ms=counter.audio_ms,
)
+ self._report_health(failure=failure, answered=answered)
def __getattr__(self, name: str) -> Any:
# ``transcribe_pcm`` is the path the live microphone really takes —
@@ -368,14 +427,22 @@ class MeteredSTT(_MeteredProvider):
"""
rate = int(args[0]) if args else int(kwargs.get("sample_rate", DEFAULT_PCM_SAMPLE_RATE))
audio_ms = pcm_duration_ms(pcm_bytes or b"", rate)
+ failure: Exception | None = None
+ answered = False
try:
- return await self._inner.transcribe_pcm(pcm_bytes, *args, **kwargs)
+ result = await self._inner.transcribe_pcm(pcm_bytes, *args, **kwargs)
+ answered = True
+ return result
+ except Exception as exc:
+ failure = exc
+ raise
finally:
self._report(
model_or_voice=self._stt_rate_key(),
chars=0,
audio_ms=audio_ms,
)
+ self._report_health(failure=failure, answered=answered)
def _stt_rate_key(self) -> str:
"""The model that priced this transcription.
diff --git a/tests/conftest.py b/tests/conftest.py
index 3bcf5331c4..e02ad9b217 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -215,6 +215,20 @@ def _app_action_state_in_tmp(tmp_path_factory, monkeypatch): # noqa: ANN001
history.reset_for_tests()
[email protected](autouse=True)
+def _provider_health_ledger_in_tmp(tmp_path_factory): # noqa: ANN001
+ """Every real brain/speech/realtime call records its outcome for the
+ status dots; keep that record off the developer's real ``data/state`` and
+ start every test with an empty one."""
+ from jarvis.brain import provider_health_ledger as ledger
+
+ root = tmp_path_factory.mktemp("provider-health")
+ ledger.set_ledger(ledger.ProviderHealthLedger(root / ledger.LEDGER_FILE_NAME))
+ yield root
+ # In-memory, not None: a late background call must not create the real file.
+ ledger.set_ledger(ledger.ProviderHealthLedger(None))
+
+
@pytest.fixture(autouse=True)
def _reset_bus():
"""Reset the global default bus before and after each test."""
diff --git a/tests/unit/brain/test_provider_health_ledger.py b/tests/unit/brain/test_provider_health_ledger.py
new file mode 100644
index 0000000000..66353e1c9d
--- /dev/null
+++ b/tests/unit/brain/test_provider_health_ledger.py
@@ -0,0 +1,231 @@
+"""The passive provider-health record behind the status dots (2026-09-30).
+
+The dots on the API-Keys tabs, sidebar, dock and chat composer no longer probe
+a provider; they read this record, which real calls and the explicit Test
+button feed. These tests pin what it keeps, what it refuses to keep, and how
+an outcome ages.
+"""
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import pytest
+
+from jarvis.brain import provider_health_ledger as ledger
+from jarvis.brain import provider_test as pt
+
+
+class _Clock:
+ def __init__(self, now: float = 1_000_000.0) -> None:
+ self.now = now
+
+ def __call__(self) -> float:
+ return self.now
+
+
+class _CountingLedger(ledger.ProviderHealthLedger):
+ """Counts disk writes so the 'no write per call' rule is checkable."""
+
+ def __init__(self, path: Path, clock: _Clock) -> None:
+ super().__init__(path, clock=clock)
+ self.writes = 0
+
+ def _persist(self, payload): # noqa: ANN001, ANN202
+ self.writes += 1
+ super()._persist(payload)
+
+
+def clock() -> _Clock:
+ return _Clock()
+
+
+def record(tmp_path: Path, clock: _Clock) -> _CountingLedger:
+ return _CountingLedger(tmp_path / "provider_health.json", clock)
+
+
+# ── what is evidence ─────────────────────────────────────────────────────────
+
+
+ "status",
+ [pt.OK, pt.BAD_KEY, pt.NO_CREDITS, pt.RATE_LIMITED, pt.MODEL_UNAVAILABLE, pt.UNREACHABLE],
+)
+def test_a_decisive_outcome_of_a_real_call_is_kept(record, status: str) -> None:
+ assert record.record("openai", ledger.MODALITY_BRAIN, status) is True
+ outcome = record.get("openai", ledger.MODALITY_BRAIN)
+ assert outcome is not None and outcome.status == status
+
+
[email protected]("status", [pt.ERROR, pt.NOT_CONFIGURED, "whatever"])
+def test_an_unrecognised_failure_of_normal_use_is_not_evidence(record, status: str) -> None:
+ """A tool bug, a cancelled turn or a missing key says nothing about the
+ provider's account — it must not paint its dot red."""
+ assert record.record("openai", ledger.MODALITY_BRAIN, status) is False
+ assert record.get("openai", ledger.MODALITY_BRAIN) is None
+
+
+def test_an_explicit_test_verdict_counts_even_when_it_is_error(record) -> None:
+ assert record.record(
+ "openai", ledger.MODALITY_BRAIN, pt.ERROR, source=ledger.SOURCE_TEST
+ )
+ assert record.get("openai", ledger.MODALITY_BRAIN).status == pt.ERROR
+
+
+def test_an_unknown_modality_is_refused(record) -> None:
+ assert record.record("openai", "vision", pt.OK) is False
+
+
+ ("error", "expected"),
+ [
+ ("Error code: 401 - invalid x-api-key", pt.BAD_KEY),
+ ("Error code: 402 - Payment Required", pt.NO_CREDITS),
+ ("429 insufficient_quota: check your plan and billing details", pt.NO_CREDITS),
+ ("HTTP 429", pt.RATE_LIMITED),
+ (TimeoutError(), pt.UNREACHABLE),
+ ],
+)
+def test_a_failure_is_classified_like_the_test_button(record, error, expected) -> None:
+ record.record_failure("grok", ledger.MODALITY_BRAIN, error)
+ assert record.get("grok", ledger.MODALITY_BRAIN).status == expected
+
+
+def test_no_error_body_or_secret_reaches_the_file(record, tmp_path: Path) -> None:
+ """AP-34: only the classification survives, never the provider's words."""
+ body = 'Error code: 401 - {"error": "Incorrect API key provided: sk-proj-SECRET123"}'
+ record.record_failure("openai", ledger.MODALITY_BRAIN, body)
+
+ text = (tmp_path / "provider_health.json").read_text(encoding="utf-8")
+ assert "SECRET123" not in text
+ assert "Incorrect" not in text
+ assert json.loads(text)["outcomes"][0]["status"] == pt.BAD_KEY
+
+
+# ── how outcomes combine and age ─────────────────────────────────────────────
+
+
+def test_a_newer_credential_failure_on_a_sibling_modality_wins(record, clock) -> None:
+ """Brain and tool model of one provider share the key."""
+ record.record("gemini", ledger.MODALITY_TOOL, pt.OK)
+ clock.now += 10
+ record.record("gemini", ledger.MODALITY_BRAIN, pt.NO_CREDITS)
+
+ assert record.effective("gemini", ledger.MODALITY_TOOL).status == pt.NO_CREDITS
+
+
+def test_a_model_failure_stays_on_its_own_modality(record, clock) -> None:
+ record.record("gemini", ledger.MODALITY_TOOL, pt.MODEL_UNAVAILABLE)
+ clock.now += 10
+ record.record("gemini", ledger.MODALITY_BRAIN, pt.OK)
+
+ assert record.effective("gemini", ledger.MODALITY_TOOL).status == pt.MODEL_UNAVAILABLE
+ assert record.effective("gemini", ledger.MODALITY_BRAIN).status == pt.OK
+
+
+def test_a_later_success_with_the_same_key_voids_an_old_credential_failure(
+ record, clock
+) -> None:
+ record.record("gemini", ledger.MODALITY_TOOL, pt.BAD_KEY)
+ clock.now += 10
+ record.record("gemini", ledger.MODALITY_BRAIN, pt.OK)
+
+ assert record.effective("gemini", ledger.MODALITY_TOOL) is None
+
+
+def test_a_transient_failure_ages_out_but_a_key_failure_does_not(record, clock) -> None:
+ record.record("grok", ledger.MODALITY_BRAIN, pt.RATE_LIMITED)
+ record.record("openai", ledger.MODALITY_BRAIN, pt.BAD_KEY)
+ clock.now += ledger.TRANSIENT_TTL_S + 1
+
+ assert record.effective("grok", ledger.MODALITY_BRAIN) is None
+ assert record.effective("openai", ledger.MODALITY_BRAIN).status == pt.BAD_KEY
+
+
+def test_the_openrouter_speech_plugin_is_filed_under_its_card(record) -> None:
+ """The plugin calls itself ``openrouter`` — the BRAIN card's id."""
+ record.record("openrouter", ledger.MODALITY_TTS, pt.OK)
+
+ assert record.get("openrouter-tts", ledger.MODALITY_TTS) is not None
+ assert record.get("openrouter", ledger.MODALITY_BRAIN) is None
+
+
+# ── forgetting ───────────────────────────────────────────────────────────────
+
+
+def test_forget_drops_every_modality_of_a_provider(record) -> None:
+ record.record("openai", ledger.MODALITY_BRAIN, pt.BAD_KEY)
+ record.record("openai", ledger.MODALITY_TOOL, pt.OK)
+ record.record("grok", ledger.MODALITY_BRAIN, pt.OK)
+
+ assert record.forget(["openai"]) == 2
+ assert record.get("openai", ledger.MODALITY_BRAIN) is None
+ assert record.get("grok", ledger.MODALITY_BRAIN) is not None
+
+
+def test_a_model_change_keeps_a_credential_failure(record) -> None:
+ record.record("openai", ledger.MODALITY_BRAIN, pt.BAD_KEY)
+ record.record("gemini", ledger.MODALITY_BRAIN, pt.MODEL_UNAVAILABLE)
+
+ record.forget(["openai", "gemini"], modality=ledger.MODALITY_BRAIN,
+ keep_credential_failures=True)
+
+ assert record.get("openai", ledger.MODALITY_BRAIN).status == pt.BAD_KEY
+ assert record.get("gemini", ledger.MODALITY_BRAIN) is None
+
+
+# ── persistence ──────────────────────────────────────────────────────────────
+
+
+def test_outcomes_survive_a_restart(record, tmp_path: Path, clock) -> None:
+ record.record("elevenlabs", ledger.MODALITY_TTS, pt.NO_CREDITS)
+
+ reborn = ledger.ProviderHealthLedger(tmp_path / "provider_health.json", clock=clock)
+
+ assert reborn.get("elevenlabs", ledger.MODALITY_TTS).status == pt.NO_CREDITS
+ assert reborn.version > 0
+
+
+def test_the_file_is_written_on_change_not_on_every_call(record, clock) -> None:
+ """The voice path records every sentence; it must pay a dict update, not I/O."""
+ for _ in range(50):
+ record.record("elevenlabs", ledger.MODALITY_TTS, pt.OK)
+ clock.now += 1
+ assert record.writes == 1
+
+ record.record("elevenlabs", ledger.MODALITY_TTS, pt.NO_CREDITS)
+ assert record.writes == 2
+
+
+def test_the_version_moves_only_when_a_status_changes(record) -> None:
+ start = record.version
+ record.record("grok", ledger.MODALITY_BRAIN, pt.OK)
+ after_first = record.version
+ record.record("grok", ledger.MODALITY_BRAIN, pt.OK)
+
+ assert after_first > start
+ assert record.version == after_first
+
+
+def test_a_damaged_file_starts_empty_instead_of_failing(tmp_path: Path) -> None:
+ path = tmp_path / "provider_health.json"
+ path.write_text("{not json", encoding="utf-8")
+
+ damaged = ledger.ProviderHealthLedger(path)
+
+ assert damaged.get("openai", ledger.MODALITY_BRAIN) is None
+ assert damaged.record("openai", ledger.MODALITY_BRAIN, pt.OK) is True
+
+
+def test_module_helpers_never_raise_into_the_call_path(monkeypatch) -> None:
+ """A health record must never cost the call it describes."""
+
+ class _Broken(ledger.ProviderHealthLedger):
+ def record(self, *args, **kwargs): # noqa: ANN002, ANN003, ANN202
+ raise OSError("disk gone")
+
+ ledger.set_ledger(_Broken(None))
+ ledger.record_success("openai", ledger.MODALITY_BRAIN)
+ ledger.record_failure("openai", ledger.MODALITY_BRAIN, RuntimeError("x"))
diff --git a/tests/unit/brain/test_run_task_fallback.py b/tests/unit/brain/test_run_task_fallback.py
index e673ae8245..dfdb268fd7 100644
--- a/tests/unit/brain/test_run_task_fallback.py
+++ b/tests/unit/brain/test_run_task_fallback.py
@@ -143,6 +143,26 @@ async def test_402_moves_on_to_the_next_family(monkeypatch: pytest.MonkeyPatch)
assert [c[0] for c in calls] == ["openrouter", "gemini"]
+async def test_scheduled_turns_feed_the_tool_model_health_record(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """The Tool Model tab's dot reads what real tool-model turns did (it no
+ longer probes): the 402 and the answering fallback are both recorded."""
+ from jarvis.brain import provider_health_ledger as ledger
+
+ mgr = _manager(monkeypatch)
+ _install_dispatch(monkeypatch, mgr, {
+ "openrouter": _Credit402("Error code: 402 - Insufficient credits"),
+ "gemini": "digest from gemini",
+ })
+
+ await mgr.run_task(prompt="p", allowed_tools=("gmail",), model_tier="fast")
+
+ record = ledger.get_ledger()
+ assert record.get("openrouter", ledger.MODALITY_TOOL).status == "no_credits"
+ assert record.get("gemini", ledger.MODALITY_TOOL).status == "ok"
+
+
async def test_the_whole_chain_is_walked_not_just_one_fallback(
monkeypatch: pytest.MonkeyPatch,
) -> None:
diff --git a/tests/unit/cu/test_brain_call_resilience.py b/tests/unit/cu/test_brain_call_resilience.py
index dd22780fb2..99add94fa5 100644
--- a/tests/unit/cu/test_brain_call_resilience.py
+++ b/tests/unit/cu/test_brain_call_resilience.py
@@ -168,6 +168,23 @@ async def test_account_failures_are_never_retried_in_place(detail: str) -> None:
assert len(alpha.requests) == 1
+async def test_tool_model_calls_feed_the_health_record() -> None:
+ """The Tool Model tab's dot reads what real CU steps did — it no longer
+ sends a probe of its own. Only the classification is kept."""
+ from jarvis.brain import provider_health_ledger as ledger
+
+ alpha = _ScriptedBrain([RuntimeError(_BAD_KEY)])
+ beta = _ScriptedBrain([(_COMPLETE, "stop")])
+ manager = _FakeManager({"alpha": alpha, "beta": beta})
+
+ reply = await _call(manager)
+
+ record = ledger.get_ledger()
+ assert reply.provider == "beta"
+ assert record.get("alpha", ledger.MODALITY_TOOL).status == "bad_key"
+ assert record.get("beta", ledger.MODALITY_TOOL).status == "ok"
+
+
async def test_exhausted_chain_still_raises_the_no_provider_error() -> None:
"""The retry must not swallow a genuinely dead chain — the engine maps this
exception to exit 3 ("no eyes — check your keys/credit")."""
diff --git a/tests/unit/speech/test_usage_meter_health.py b/tests/unit/speech/test_usage_meter_health.py
new file mode 100644
index 0000000000..b6eb001a79
--- /dev/null
+++ b/tests/unit/speech/test_usage_meter_health.py
@@ -0,0 +1,176 @@
+"""Real speech calls feed the provider-health record behind the status dots.
+
+The Voice Output / Voice Input tabs used to learn whether a key worked by
+synthesising "Test." and sending half a second of silence to the cloud
+recognizer on every app start and reload. They now read what real speech
+calls did, and the speech meter — which already wraps every TTS/STT call —
+is where those outcomes are reported.
+"""
+from __future__ import annotations
+
+import asyncio
+from collections.abc import AsyncIterator
+
+import pytest
+
+from jarvis.brain import provider_health_ledger as ledger
+from jarvis.brain import provider_test as pt
+from jarvis.core.protocols import AudioChunk
+from jarvis.plugins.tts.fallback_tts import FallbackTTS
+from jarvis.speech.stt_fallback import FallbackSTT
+from jarvis.speech.usage_meter import SpeechUsage, meter_stt, meter_tts
+
+_PCM = b"\x00" * 3200 # 100 ms of 16 kHz mono int16
+
+
+class _Sink:
+ def record(self, usage: SpeechUsage) -> None:
+ pass
+
+
+class _TTS:
+ """A TTS plugin; ``fail`` raises before audio, ``fell_back_to`` mimics a
+ plugin that absorbed its own error and let another family speak."""
+
+ supports_streaming = True
+
+ def __init__(
+ self,
+ name: str = "elevenlabs",
+ *,
+ fail: Exception | None = None,
+ fell_back_to: str | None = None,
+ absorbed: str | None = None,
+ ) -> None:
+ self.name = name
+ self._fail = fail
+ self._fell_back_to = fell_back_to
+ self._absorbed = absorbed
+ self.last_voice_provider: str | None = None
+ self.last_failure: str | None = None
+
+ async def synthesize(
+ self, text: str, voice: str | None = None, language_code: str | None = None
+ ) -> AsyncIterator[AudioChunk]:
+ self.last_voice_provider = self.name
+ self.last_failure = None
+ if self._fail is not None:
+ raise self._fail
+ if self._fell_back_to:
+ self.last_failure = self._absorbed
+ self.last_voice_provider = self._fell_back_to
+ yield AudioChunk(pcm=_PCM, sample_rate=16_000, timestamp_ns=0)
+
+
+class _STT:
+ supports_streaming = False
+
+ def __init__(self, name: str = "groq-api", *, fail: Exception | None = None) -> None:
+ self.name = name
+ self._fail = fail
+
+ async def transcribe_pcm(self, pcm: bytes, sample_rate: int = 16_000, **_kw) -> str:
+ if self._fail is not None:
+ raise self._fail
+ return "hello"
+
+
+async def _speak(tts) -> int: # noqa: ANN001
+ produced = 0
+ async for _chunk in tts.synthesize("Hello there."):
+ produced += 1
+ return produced
+
+
+def _status(provider: str, modality: str) -> str | None:
+ outcome = ledger.get_ledger().get(provider, modality)
+ return outcome.status if outcome is not None else None
+
+
+def test_a_spoken_sentence_records_the_voice_as_answering() -> None:
+ asyncio.run(_speak(meter_tts(_TTS(), _Sink())))
+
+ assert _status("elevenlabs", ledger.MODALITY_TTS) == pt.OK
+
+
+def test_a_failing_synthesis_records_its_class_and_still_raises() -> None:
+ tts = meter_tts(_TTS(fail=RuntimeError("HTTP 401 Unauthorized")), _Sink())
+
+ with pytest.raises(RuntimeError):
+ asyncio.run(_speak(tts))
+
+ assert _status("elevenlabs", ledger.MODALITY_TTS) == pt.BAD_KEY
+
+
+def test_an_absorbed_failure_is_charged_to_the_voice_that_was_asked() -> None:
+ """The plugin crossed to another family and the user heard a different
+ voice; the asked provider's key problem must still reach its dot."""
+ tts = _TTS(fell_back_to="gemini-flash-tts", absorbed="_GrokFatalError: HTTP 402")
+
+ asyncio.run(_speak(meter_tts(tts, _Sink())))
+
+ assert _status("elevenlabs", ledger.MODALITY_TTS) == pt.NO_CREDITS
+ assert _status("gemini-flash-tts", ledger.MODALITY_TTS) == pt.OK
+
+
+class _SlowTTS(_TTS):
+ async def synthesize(
+ self, text: str, voice: str | None = None, language_code: str | None = None
+ ) -> AsyncIterator[AudioChunk]:
+ await asyncio.Event().wait() # the barge-in comes first
+ yield AudioChunk(pcm=_PCM, sample_rate=16_000, timestamp_ns=0)
+
+
+def test_a_sentence_cancelled_before_audio_is_no_evidence() -> None:
+ async def _barge_in() -> None:
+ task = asyncio.create_task(_speak(meter_tts(_SlowTTS(), _Sink())))
+ await asyncio.sleep(0.01)
+ task.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await task
+
+ asyncio.run(_barge_in())
+
+ assert _status("elevenlabs", ledger.MODALITY_TTS) is None
+
+
+def test_a_transcription_records_the_recognizer() -> None:
+ stt = meter_stt(_STT(), _Sink())
+
+ assert asyncio.run(stt.transcribe_pcm(_PCM, 16_000)) == "hello"
+ assert _status("groq-api", ledger.MODALITY_STT) == pt.OK
+
+
+def test_a_failing_transcription_records_its_class() -> None:
+ stt = meter_stt(_STT(fail=RuntimeError("429 Too Many Requests")), _Sink())
+
+ with pytest.raises(RuntimeError):
+ asyncio.run(stt.transcribe_pcm(_PCM, 16_000))
+
+ assert _status("groq-api", ledger.MODALITY_STT) == pt.RATE_LIMITED
+
+
+def test_an_stt_crossover_charges_the_primary_and_credits_the_fallback() -> None:
+ primary = _STT("groq-api", fail=RuntimeError("Error code: 429 - rate limit exceeded"))
+ chain = FallbackSTT(
+ primary,
+ ["openai-api"],
+ lambda name: _STT(name),
+ primary_name="groq-api",
+ )
+ stt = meter_stt(chain, _Sink())
+
+ assert asyncio.run(stt.transcribe_pcm(_PCM, 16_000)) == "hello"
+ assert chain.last_failure is not None
+ assert _status("groq-api", ledger.MODALITY_STT) == pt.RATE_LIMITED
+ assert _status("openai-api", ledger.MODALITY_STT) == pt.OK
+
+
+def test_a_tts_fallback_wrapper_exposes_what_the_primary_failed_with() -> None:
+ wrapper = FallbackTTS(_TTS("cartesia", fail=RuntimeError("HTTP 401")), _TTS("piper-local"))
+
+ asyncio.run(_speak(meter_tts(wrapper, _Sink())))
+
+ assert wrapper.last_failure is not None
+ assert _status("cartesia", ledger.MODALITY_TTS) == pt.BAD_KEY
+ assert _status("piper-local", ledger.MODALITY_TTS) == pt.OK