Skip to content

Dodaios is an open-source agent OS for your desktop: it runs your coding agents, browser and apps by voice or chat, on your own machine. Formerly Personal Jarvis.

fix(providers): status dots read the health record, never a billed probe

rubenluetke10-beep7f6ec9c7b710
diff --git a/jarvis/ui/web/agent_chat_routes.py b/jarvis/ui/web/agent_chat_routes.py
index 96a88583af..e85745d1de 100644
--- a/jarvis/ui/web/agent_chat_routes.py
+++ b/jarvis/ui/web/agent_chat_routes.py
@@ -433,19 +433,18 @@ class ProviderHealthResponse(BaseModel):
cached: bool = False
-#: How long one sweep stands. A key does not go bad between two clicks, and
-#: the sweep costs one real request per provider — 9 s to 16 s each on the
-#: maintainer's box (measured 2026-08-26), which is why nothing waits for it.
+#: How long one sweep stands. The sweep sends nothing to any provider (it
+#: reads key presence, CLI login state and the passive record of real calls),
+#: but the CLI login reads are not free, so a sweep is reused for a while — and
+#: dropped the moment the health record changes (see ``get_provider_health``).
_HEALTH_TTL_S: Final[float] = 300.0
-#: The whole sweep's ceiling. A provider still thinking when this runs out is
-#: reported ``unknown`` and draws no dot: a seat that cannot answer a one-token
-#: request inside this is not a seat someone should be told is fine, but
-#: neither is a slow network proof that a key is bad.
+#: The whole sweep's ceiling. A row whose (local) check has not finished when
+#: this runs out is reported ``unknown`` and draws no dot.
_HEALTH_SWEEP_S: Final[float] = 20.0
-#: surface -> (checked_at, rows). Process-local, like every other short cache
-#: in the routes; a restart simply re-sweeps on first use.
-_health_cache: dict[str, tuple[float, list[ProviderHealthRow]]] = {}
+#: surface -> (checked_at, health-record version, rows). Process-local, like
+#: every other short cache in the routes; a restart simply re-sweeps.
+_health_cache: dict[str, tuple[float, int, list[ProviderHealthRow]]] = {}
#: Vendor CLI runners. Their credential is a subscription login, not a key,
#: so the API-Keys one-token probe is the wrong check: the dual Claude row
@@ -560,7 +559,7 @@ async def _one_provider_health(cfg: Any, provider_id: str, *, surface: str) -> P
from jarvis.ui.web.provider_routes import provider_health
try:
- health = await provider_health(cfg, provider_id, probe=True)
+ health = await provider_health(cfg, provider_id)
except TimeoutError:
return ProviderHealthRow(
provider=provider_id, status="unknown", reason="timeout", detail="No answer in time"
@@ -593,22 +592,33 @@ async def get_provider_health(
2026-08-26, four of nine connected rows were in one of those states, and
the picker offered all nine as if they were equal.
- API / brain seats run the real one-token check the API-Keys screen's tab
- dots use (``provider_routes.provider_health``). A vendor CLI seat does
- not: it spends a subscription login, so this reports that login instead.
- The dual Claude row is why the split exists — its catalog id is
- ``claude-api``, and the API-Keys probe would otherwise paint "Key
- rejected" on a Claude Code seat that is signed in.
+ API / brain seats report what the API-Keys screen's tab dots report
+ (``provider_routes.provider_health``): key presence plus the outcome of
+ that provider's last REAL call. Nothing here sends a request — until
+ 2026-09-30 every open of the chat spent a paid one-token completion per
+ keyed provider. A seat never used since its key was saved is ``unknown``
+ and draws no dot. A vendor CLI seat spends a subscription login, so this
+ reports that login instead. The dual Claude row is why the split exists —
+ its catalog id is ``claude-api``, and the API-key verdict would otherwise
+ paint "Key rejected" on a Claude Code seat that is signed in.
Nothing waits for this: the composer paints from the catalog and folds
these in when they land. A row that does not finish inside the sweep
ceiling comes back ``unknown`` and is drawn exactly as it was before.
"""
_service(request) # 503 like every other route when the chat is off
+ from jarvis.brain.provider_health_ledger import ledger_version
+
now = time.monotonic()
+ version = ledger_version()
cached = _health_cache.get(surface)
- if cached and not refresh and (now - cached[0]) < _HEALTH_TTL_S:
- return ProviderHealthResponse(providers=cached[1], checked_at=cached[0], cached=True)
+ if (
+ cached
+ and not refresh
+ and cached[1] == version
+ and (now - cached[0]) < _HEALTH_TTL_S
+ ):
+ return ProviderHealthResponse(providers=cached[2], checked_at=cached[0], cached=True)
from jarvis.ui.web.provider_routes import _resolve_cfg
@@ -636,7 +646,7 @@ async def get_provider_health(
provider=pid, status="unknown", reason="timeout", detail="No answer in time"
)
)
- _health_cache[surface] = (now, rows)
+ _health_cache[surface] = (now, version, rows)
return ProviderHealthResponse(providers=rows, checked_at=now, cached=False)
diff --git a/jarvis/ui/web/provider_routes.py b/jarvis/ui/web/provider_routes.py
index 205ce1eca6..dc7f474449 100644
--- a/jarvis/ui/web/provider_routes.py
+++ b/jarvis/ui/web/provider_routes.py
@@ -28,6 +28,7 @@ from typing import Any, Final, Literal, get_args
from fastapi import APIRouter, HTTPException, Request, Response
from pydantic import BaseModel, Field
+from jarvis.brain import provider_health_ledger as _health_ledger
from jarvis.brain import provider_test as _provider_test
from jarvis.brain import section_health as _section_health
from jarvis.brain.model_catalog import ModelInfo, catalog_spec, classify_model
@@ -1385,8 +1386,11 @@ async def test_provider_connection(provider_id: str, request: Request) -> Provid
detail=(f"Test timed out after {ceiling:.0f}s — the provider did not answer."),
latency_ms=ceiling * 1000.0,
)
- # This exact result is newer than any overlapping section sweep. Cancel the
- # older snapshot so the UI refresh cannot resurrect a pre-test status.
+ # The status dots read the passive health record, never a probe of their
+ # own; this user-clicked verdict is the freshest evidence there is.
+ _record_test_verdict(spec, result.status)
+ # This exact result is newer than any cached rollup. Drop it so the UI
+ # refresh reads the verdict just recorded.
_invalidate_section_health_state(request)
return ProviderTestResponse(
provider=result.provider,
@@ -1410,13 +1414,19 @@ assert set(get_args(SectionHealthStatusLiteral)) == set(_section_health.SECTION_
"section-health status vocabulary drift (Pydantic Literal vs SSOT)"
)
-# Cache the rollup so opening the API-Keys page / switching tabs does not re-run
-# the REAL connectivity tests on every render. ``?refresh=true`` (used by the UI
-# after a key save / provider switch) bypasses it. The sidebar and dock mount
-# this poll in every window, and every frontend build reloads every window, so a
-# short TTL turned into a paid probe per provider about once a minute (live
-# 2026-09-29: the voice key ran dry on health checks). Selection changes still
-# re-check at once through the fingerprint and the explicit refresh.
+# The rollup sends NOTHING to any provider (decision 2026-09-30): it reads
+# credential presence, a CLI's local login state, the on-disk install state of
+# a local model and the passive health record (``provider_health_ledger``) that
+# real calls and the explicit Test button feed. It used to run a real
+# completion, a real TTS synthesis, cloud STT audio and a billed realtime
+# session start per tier, on every app start, window reload and key/provider
+# event — an idle app spent money on its own health dots.
+#
+# What is left to cache is the cheap part (keyring reads, CLI login status), so
+# every window's sidebar and dock can mount this without repeating it. The
+# fingerprint carries the ledger version, so a newly recorded outcome, a
+# selection change or a key save (which drops the cache) is visible on the very
+# next read; ``?refresh=true`` still recomputes without the cache.
_SECTION_HEALTH_TTL_S = 30 * 60.0
_SECTION_HEALTH_KEYS = (
"brain",
@@ -1429,6 +1439,11 @@ _SECTION_HEALTH_KEYS = (
"advanced",
)
+#: ``reason`` of a keyed provider whose credential is present but which has not
+#: made a real call since (status ``unknown``: no dot). Readiness counts it as
+#: set up — it will be checked by its first real use, never by a probe.
+UNVERIFIED_REASON: Final[str] = "unverified"
+
class SectionHealth(BaseModel):
"""One tab's rolled-up health. Only ``needs_setup`` (amber) and ``error``
@@ -1453,13 +1468,63 @@ class SectionHealthResponse(BaseModel):
cached: bool = False
+#: What the last real outcome means, as the tail of a card/tooltip sentence.
+#: Deliberately our own words: a provider's error body never reaches the UI,
+#: a log line or the record (AP-34).
+_OUTCOME_PHRASES: dict[str, str] = {
+ _provider_test.OK: "answered",
+ _provider_test.BAD_KEY: "the key was rejected",
+ _provider_test.NO_CREDITS: "out of credit or quota",
+ _provider_test.RATE_LIMITED: "rate-limited",
+ _provider_test.MODEL_UNAVAILABLE: "the selected model was not available",
+ _provider_test.UNREACHABLE: "did not answer in time",
+ _provider_test.ERROR: "the check failed",
+}
+
+
+def _age_phrase(seconds: float) -> str:
+ if seconds < 90:
+ return "just now"
+ minutes = int(seconds // 60)
+ if minutes < 90:
+ return f"{minutes} min ago"
+ hours = int(seconds // 3600)
+ if hours < 48:
+ return f"{hours} h ago"
+ return f"{int(seconds // 86400)} d ago"
+
+
+def _outcome_detail(label: str, outcome: _health_ledger.Outcome) -> str:
+ """``"<label>: <what happened> on its last real call (<age>)"``."""
+ what = _OUTCOME_PHRASES.get(outcome.status, outcome.status)
+ where = "on the last test" if outcome.source == _health_ledger.SOURCE_TEST else (
+ "on its last real call"
+ )
+ age = _age_phrase(max(0.0, time.time() - outcome.at))
+ return f"{label}: {what} {where} ({age})"
+
+
+def _record_test_verdict(spec: ProviderSpec, status: str, *, model: str | None = None) -> None:
+ """Feed an explicit test's verdict into the passive health record.
+
+ Only for the checks a person asked for — the card's Test button, the check
+ right after a key is saved, the probe of a model they just picked. The
+ rollup and the composer never probe; they read what this left behind.
+ """
+ modality = _health_ledger.modality_for_tier(getattr(spec, "tier", None))
+ if modality is None:
+ return
+ _health_ledger.record_status(
+ spec.id, modality, status, source=_health_ledger.SOURCE_TEST, model=model
+ )
+
+
async def _tier_section_health(
cfg: Any,
spec: ProviderSpec | None,
*,
- model: str | None = None,
+ modality: str | None = None,
optional: bool = False,
- probe: bool = True,
binary_path: str | None = None,
) -> SectionHealth:
"""Health of one provider tier, derived from its ACTIVE provider only.
@@ -1468,9 +1533,20 @@ async def _tier_section_health(
deliberately NOT "does any provider here lack a key" (that would paint every
tab red, since unused providers are normally left empty).
- ``model`` probes that exact model for a brain-tier spec — used by sections
- whose tier carries its own model pin (Tool Model), so the dot reflects what
- that tier actually runs, not the general brain model.
+ Nothing here calls a provider (2026-09-30). The verdict is built from:
+
+ * credential presence (a stored key, or a CLI's local login state);
+ * for an on-device card, whether its engine and weights are on disk;
+ * the passive health record for ``modality`` — the last REAL outcome of this
+ provider (a real brain turn, tool step, spoken sentence, transcription or
+ realtime handshake), or the verdict of an explicit Test.
+
+ A keyed provider that has not been used since its key was saved is
+ ``unknown`` / ``unverified``: silent, never red — it is checked by its first
+ real use. A login-based provider (Codex, Antigravity, Grok Build, Claude
+ Code) is ``ok`` on its login alone, which is the whole verdict the Test
+ button gives for it too. ``modality=None`` judges on credential presence
+ alone (the dictation wording pass, whose key another tier owns).
``optional`` marks a tier the install does not depend on. A missing key then
reports ``ok``/``not_configured_optional`` instead of amber ``needs_setup``,
@@ -1481,11 +1557,6 @@ async def _tier_section_health(
that must stay silent); a spec that carries ``optional`` itself also counts.
An optional tier still turns RED when a key IS present and failing — the
rule suppresses nagging, never a real fault.
-
- ``probe=False`` skips the live provider call and reports on credential
- presence alone. For a tier whose credential is a key another tier already
- owns and tests, a second network round-trip on every page open buys no
- signal it does not already have.
"""
optional = optional or bool(getattr(spec, "optional", False))
if spec is None:
@@ -1506,8 +1577,7 @@ async def _tier_section_health(
# same as "usable", and reading it that way is how an on-device card comes to
# claim it works on a machine where its engine and weights were never
# installed. So ask the disk (a cheap file check, no model load) before
- # calling the tier healthy. The real inference test still stays off the
- # page-open path: it would force a multi-gigabyte load for no extra signal.
+ # calling the tier healthy.
if getattr(spec, "auth_mode", None) == "none":
local_state = await asyncio.to_thread(_local_runtime_payload, spec)
if local_state is not None and not local_state["ready"]:
@@ -1528,7 +1598,8 @@ async def _tier_section_health(
spec,
binary_path,
)
- except Exception: # noqa: BLE001 — a probe failure is "not set up", not a crash
+ except Exception as exc: # noqa: BLE001 — a failed presence check is "not set up", not a crash
+ log.info("section-health credential check for %s failed: %s", spec.id, exc)
configured = False
if not configured:
if optional:
@@ -1544,41 +1615,50 @@ async def _tier_section_health(
detail=f"{spec.label}: not connected or configured",
subject_id=spec.id,
)
- if not probe:
+ if modality is None:
return SectionHealth(
status=_section_health.OK,
reason="configured",
detail=f"{spec.label}: connected or configured",
subject_id=spec.id,
)
- try:
- result = await _run_tier_test(spec, cfg, model=model)
- except Exception as exc: # noqa: BLE001
- log.warning("section-health test for %s failed: %s", spec.id, exc)
+ outcome = _health_ledger.effective_outcome(spec.id, modality)
+ if outcome is None:
+ if getattr(spec, "auth_mode", None) != "api_key":
+ return SectionHealth(
+ status=_section_health.OK,
+ reason="connected",
+ detail=f"{spec.label}: signed in",
+ subject_id=spec.id,
+ )
return SectionHealth(
status=_section_health.UNKNOWN,
- reason="error",
- detail=f"{spec.label}: check failed",
+ reason=UNVERIFIED_REASON,
+ detail=f"{spec.label}: connected, checked on its first real use",
subject_id=spec.id,
)
- status = _section_health.section_status_for_test(result.status, configured=True)
return SectionHealth(
- status=status,
- reason=result.status,
- detail=f"{spec.label}: {_human_detail(result) or result.status}",
+ status=_section_health.section_status_for_test(outcome.status, configured=True),
+ reason=outcome.status,
+ detail=_outcome_detail(spec.label, outcome),
subject_id=spec.id,
)
-async def provider_health(cfg: Any, provider_id: str, *, probe: bool = True) -> SectionHealth:
- """The live health of ONE named provider, for a caller outside this module.
+async def provider_health(cfg: Any, provider_id: str) -> SectionHealth:
+ """The health of ONE named provider, for a caller outside this module.
- The same check the API-Keys tabs run, addressed by provider id instead of
- by tier: a credential that is present but rejected is ``error`` with the
- reason ("bad_key", "no_credits", …), a missing one is ``needs_setup``, a
- working one is ``ok``. The agent chat's composer uses it to say which
+ The same verdict the API-Keys tabs show, addressed by provider id instead
+ of by tier: a credential whose last real call was rejected is ``error``
+ with the reason ("bad_key", "no_credits", …), a missing one is
+ ``needs_setup``, one that answered is ``ok``, one never used since its key
+ was saved is ``unknown``. The agent chat's composer uses it to say which
seats actually answer, so the two surfaces can never disagree about
- whether a provider works — there is one check, not two opinions.
+ whether a provider works — there is one record, not two opinions.
+
+ Sends nothing to the provider: it reads the passive health record
+ (2026-09-30 — every open of the chat used to spend a paid completion per
+ keyed provider).
An unknown id is ``unknown`` rather than an error: a caller listing rows
must not lose its whole answer to one name this module has never heard of.
@@ -1591,13 +1671,9 @@ async def provider_health(cfg: Any, provider_id: str, *, probe: bool = True) ->
detail=f"{provider_id}: not a known provider",
subject_id=provider_id,
)
- # The realtime voice key is probed by the Realtime tab only: a composer
- # sweep that spent it again on every open drained it (mandate 2026-09-29).
- from jarvis.brain.voice_key import bills_voice_key
-
- if bills_voice_key(cfg, provider_id):
- probe = False
- return await _tier_section_health(cfg, spec, probe=probe)
+ return await _tier_section_health(
+ cfg, spec, modality=_health_ledger.modality_for_tier(spec.tier)
+ )
def _worker_usable(provider: str) -> bool:
@@ -1768,13 +1844,16 @@ async def _realtime_section_health(
*,
binary_path: str | None = None,
) -> SectionHealth:
- """Test the active provider's actual duplex handshake.
+ """The active realtime provider, judged by its last REAL handshake.
- Credential presence alone previously painted a depleted or schema-broken
- provider green. Reuse the standard tier health mapping so the Realtime tab
- reports the same honest account/integration states as every other tier.
+ Credential presence alone once painted a depleted or schema-broken provider
+ green, so the tab reports the outcome of the last real session start (or an
+ explicit Test) — never a session of its own: opening one is a billed
+ mutation, and the rollup used to open one on every app start and reload.
"""
- return await _tier_section_health(cfg, spec, binary_path=binary_path)
+ return await _tier_section_health(
+ cfg, spec, modality=_health_ledger.MODALITY_REALTIME, binary_path=binary_path
+ )
async def _dictation_section_health(
@@ -1791,10 +1870,9 @@ async def _dictation_section_health(
amber dot on every install forever. That dot would be the feature's most
visible effect on the majority of users, which is the opposite of what an
optional convenience should do.
- * ``probe=False`` — no live call. The credential here is always a key some
- other tier already owns and tests (the Groq speech-to-text key, the
- Gemini/OpenAI/OpenRouter brain key), so probing again on page open would
- duplicate a request without producing a signal we do not already have.
+ * no health record (``modality=None``) — credential presence decides. The
+ credential here is always a key some other tier already owns and reports
+ on (the Groq speech-to-text key, the Gemini/OpenAI/OpenRouter brain key).
``enabled`` is reported separately from "no key", because "you switched it
off" and "you have no key for it" are different answers to "why is nothing
@@ -1807,7 +1885,7 @@ async def _dictation_section_health(
detail="Dictation polish is switched off",
subject_id=None,
)
- return await _tier_section_health(cfg, spec, optional=True, probe=False)
+ return await _tier_section_health(cfg, spec, optional=True)
def _advanced_section_health(request: Request) -> SectionHealth:
@@ -1895,6 +1973,8 @@ def _section_health_fingerprint(
("dictation-polish", "1" if _polish_enabled(cfg) else "0"),
("dictation-provider", str(getattr(dictation, "polish_provider", None) or "")),
("advanced-reachable", repr(reachable)),
+ # A newly recorded real-call outcome supersedes the cached rollup.
+ ("health-ledger", str(_health_ledger.ledger_version())),
)
return tuple((key, subjects.get(key) or "") for key in _SECTION_HEALTH_KEYS) + configuration
@@ -2041,29 +2121,34 @@ async def _compute_section_health(
)
binary_path = _codex_binary_path(request)
- # The realtime section already probes the voice key. A brain on that same
- # key would spend it a second time for the same answer, so it reports from
- # configuration only.
- from jarvis.brain.voice_key import bills_voice_key
-
+ # Every section reads the passive health record; none sends a request.
checks = {
"brain": _tier_section_health(
cfg,
get_spec(subjects["brain"] or ""),
- probe=not bills_voice_key(cfg, subjects["brain"]),
+ modality=_health_ledger.MODALITY_BRAIN,
binary_path=binary_path,
),
- # The Tool Model tier has its own model pin (tool_model → cu_model);
- # probe THAT model, not the general brain model. An unset pin falls
- # through to run_provider_test's own resolution (model → tier default).
+ # The Tool Model tier runs its own model pin (tool_model → cu_model), so
+ # it is judged by the outcomes of tool-model calls, not brain turns.
"computer-use": _tier_section_health(
cfg,
get_spec(subjects["computer-use"] or ""),
- model=_provider_cu_model(cfg, subjects["computer-use"] or "") or None,
+ modality=_health_ledger.MODALITY_TOOL,
+ binary_path=binary_path,
+ ),
+ "tts": _tier_section_health(
+ cfg,
+ get_spec(subjects["tts"] or ""),
+ modality=_health_ledger.MODALITY_TTS,
+ binary_path=binary_path,
+ ),
+ "stt": _tier_section_health(
+ cfg,
+ get_spec(subjects["stt"] or ""),
+ modality=_health_ledger.MODALITY_STT,
binary_path=binary_path,
),
- "tts": _tier_section_health(cfg, get_spec(subjects["tts"] or ""), binary_path=binary_path),
- "stt": _tier_section_health(cfg, get_spec(subjects["stt"] or ""), binary_path=binary_path),
"realtime": _realtime_section_health(
cfg,
get_spec(subjects["realtime"] or ""),
@@ -2420,10 +2505,22 @@ async def _apply_brain_model(
applied_live = False
restart_required = brain is None
+ # What the previous model did says nothing about this one; whether the KEY
+ # works still holds.
+ _health_ledger.forget_providers(
+ [provider_id],
+ modality=_health_ledger.MODALITY_BRAIN,
+ keep_credential_failures=True,
+ )
probe_payload: BrainModelProbe | None = None
if probe:
probe_model = model or _current_brain_model(cfg, provider_id)
result = await _probe_brain_model(provider_id, probe_model)
+ picked_spec = get_spec(provider_id)
+ if picked_spec is not None:
+ # The user just picked this model: the probe is their action and
+ # its verdict feeds the same record the status dots read.
+ _record_test_verdict(picked_spec, result.status, model=probe_model)
probe_payload = BrainModelProbe(
status=result.status,
detail=_human_detail(result),
@@ -2505,6 +2602,9 @@ def _apply_tts_selection(
except Exception as exc: # noqa: BLE001
log.error("TTS live re-apply for %s failed: %s", provider_id, exc, exc_info=True)
+ _health_ledger.forget_providers(
+ [provider_id], modality=_health_ledger.MODALITY_TTS, keep_credential_failures=True
+ )
_invalidate_section_health_state(request)
return BrainModelSaveResponse(
ok=True,
@@ -2565,6 +2665,9 @@ def _apply_stt_model(
except Exception as exc: # noqa: BLE001
log.debug("In-memory stt model update skipped: %s", exc)
+ _health_ledger.forget_providers(
+ [provider_id], modality=_health_ledger.MODALITY_STT, keep_credential_failures=True
+ )
_invalidate_section_health_state(request)
return BrainModelSaveResponse(
ok=True,
@@ -3402,6 +3505,9 @@ async def set_cu_model(provider_id: str, body: CuModelBody, request: Request) ->
SecretConfigured(key=f"brain.providers.{provider_id}.cu_model", action="set"),
)
effective = value or _current_brain_model(cfg, provider_id)
+ _health_ledger.forget_providers(
+ [provider_id], modality=_health_ledger.MODALITY_TOOL, keep_credential_failures=True
+ )
_invalidate_section_health_state(request)
return CuModelResponse(
ok=True,
@@ -3565,6 +3671,13 @@ async def set_realtime_options(
selected_provider = str(
getattr(getattr(getattr(cfg, "brain", None), "realtime", None), "provider", "") or ""
)
+ if model is not None or voice is not None:
+ # The last handshake was for the old model/voice; the key still stands.
+ _health_ledger.forget_providers(
+ [provider_id],
+ modality=_health_ledger.MODALITY_REALTIME,
+ keep_credential_failures=True,
+ )
session_restarted = False
if selected_provider == provider_id:
from jarvis.ui.web.voice_runtime import reconnect_realtime
@@ -4043,6 +4156,19 @@ def _secret_slot_labels(slots: tuple[str, ...]) -> list[str]:
return list(labels)
+def _providers_reading_slots(slots: tuple[str, ...] | list[str]) -> list[str]:
+ """Every provider card that reads any of ``slots`` — its own key slots and
+ its resolution chain (a realtime card falls back to the family key)."""
+ wanted = {slot for slot in slots if slot}
+ found: list[str] = []
+ for spec in PROVIDERS:
+ reads = set(spec.secret_keys)
+ reads.update(slot for slot, _env in cfg_mod.PROVIDER_SECRET_CANDIDATES.get(spec.id, ()))
+ if reads & wanted:
+ found.append(spec.id)
+ return found
+
+
@router.post("/secrets/{key}", openapi_extra={"x-jarvis-dangerous": True})
async def set_secret_value(key: str, body: SecretBody, request: Request) -> dict[str, Any]:
"""Save a key for its whole provider family, or ask which scope was meant.
@@ -4081,6 +4207,8 @@ async def set_secret_value(key: str, body: SecretBody, request: Request) -> dict
slot,
)
touched = (*plan.writes, *plan.deletes)
+ # What the old key did says nothing about the new one.
+ _health_ledger.forget_providers(_providers_reading_slots(touched))
for slot in plan.writes:
await _emit(request, SecretConfigured(key=slot, action="set"))
for slot in plan.deletes:
@@ -4108,6 +4236,7 @@ async def delete_secret_value(key: str, request: Request) -> dict[str, Any]:
"credential store may be locked or unavailable."
),
)
+ _health_ledger.forget_providers(_providers_reading_slots((key,)))
await _emit(request, SecretConfigured(key=key, action="delete"))
_invalidate_section_health_state(request)
return {"ok": True, "key": key}
diff --git a/jarvis/ui/web/starter_plan_routes.py b/jarvis/ui/web/starter_plan_routes.py
index 507c94bfc4..acd5e1eb9b 100644
--- a/jarvis/ui/web/starter_plan_routes.py
+++ b/jarvis/ui/web/starter_plan_routes.py
@@ -8,9 +8,12 @@ shares the ``/api/setup`` prefix.
GET /api/setup/readiness → is the active voice mode fully set up?
POST /api/setup/readiness/celebrated → the one-time "all set" note was shown
-Readiness reuses the section-health rollup (the real per-tier probes) instead
-of a second opinion: "ready" means every section the mode needs reports
-``ok``. Reads fail open — a broken probe reports not-ready, never a 5xx.
+Readiness reuses the section-health rollup instead of a second opinion. That
+rollup sends nothing to any provider: it reads credential presence and the
+passive record of real calls. "Ready" means every section the mode needs is
+``ok``, or set up but not used yet (``unknown`` / ``unverified`` — its first
+real call checks it). A section with a known failure is never ready. Reads
+fail open — a broken check reports not-ready, never a 5xx.
"""
from __future__ import annotations
@@ -101,6 +104,21 @@ def _voice_mode(request: Request) -> str:
return mode if mode in ("pipeline", "realtime") else "realtime"
+def _section_ready(entry: dict[str, Any]) -> bool:
+ """``ok``, or a keyed provider that is set up and simply not used yet.
+
+ The rollup never probes (it would bill the key), so "configured, no real
+ call since" is ``unknown`` with reason ``unverified``. Treating that as
+ not-ready would hold the note back until every tier had been used once.
+ """
+ from .provider_routes import UNVERIFIED_REASON
+
+ status = entry.get("status")
+ if status == "ok":
+ return True
+ return status == "unknown" and entry.get("reason") == UNVERIFIED_REASON
+
+
@router.get("/readiness")
async def readiness(request: Request, refresh: bool = False) -> dict[str, Any]:
"""Whether every section the active voice mode needs answers ``ok``.
@@ -131,7 +149,7 @@ async def readiness(request: Request, refresh: bool = False) -> dict[str, Any]:
sections = {
name: {"status": "unknown", "reason": "error", "detail": ""} for name in required
}
- ready = bool(required) and all(sections[name].get("status") == "ok" for name in required)
+ ready = bool(required) and all(_section_ready(sections[name]) for name in required)
return {
"mode": mode,
"required": list(required),
diff --git a/tests/fakes/fake_provider_calls.py b/tests/fakes/fake_provider_calls.py
new file mode 100644
index 0000000000..71cead5dbc
--- /dev/null
+++ b/tests/fakes/fake_provider_calls.py
@@ -0,0 +1,80 @@
+"""Records every seam through which a health check could reach a provider.
+
+The status dots (API-Keys tabs, sidebar, dock, chat composer) must never send
+a billed request on their own (decision 2026-09-30). This fake replaces each
+seam that talks to a provider — the shared connectivity test and its default
+builders, the brain health checker, the model-picker probe, the dictation
+wording probe and brain-plugin instantiation — with a recorder. A test then
+drives the health routes and asserts :attr:`ProviderCallRecorder.calls` is
+empty; a regression that reintroduces a probe shows up as a named call
+instead of a real request to a provider.
+"""
+from __future__ import annotations
+
+from types import SimpleNamespace
+from typing import Any
+
+
+class ProviderCallRecorder:
+ """Install with :meth:`install`; inspect :attr:`calls` afterwards."""
+
+ def __init__(self) -> None:
+ self.calls: list[tuple[str, str]] = []
+
+ def _note(self, seam: str, subject: Any) -> None:
+ name = getattr(subject, "id", None) or getattr(subject, "name", None) or subject
+ self.calls.append((seam, str(name)))
+
+ def install(self, monkeypatch: Any) -> ProviderCallRecorder:
+ from jarvis.brain import healthcheck, provider_registry, provider_test
+ from jarvis.dictation import polish_probe
+ from jarvis.ui.web import provider_routes
+
+ recorder = self
+
+ async def run_provider_test(spec: Any, cfg: Any, **_kwargs: Any) -> Any:
+ recorder._note("run_provider_test", spec)
+ return SimpleNamespace(provider=getattr(spec, "id", ""), status="ok", detail="")
+
+ async def realtime_probe(spec: Any, cfg: Any, **_kwargs: Any) -> float:
+ recorder._note("realtime_probe", spec)
+ return 0.0
+
+ def make_tts(cfg: Any, provider: str) -> Any:
+ recorder._note("make_tts", provider)
+ raise AssertionError("a health check built a TTS provider")
+
+ def make_stt(cfg: Any, provider: str) -> Any:
+ recorder._note("make_stt", provider)
+ raise AssertionError("a health check built an STT provider")
+
+ async def checker_probe(_self: Any, provider: str, model: str, **_kwargs: Any) -> Any:
+ recorder._note("brain_health_probe", provider)
+ return SimpleNamespace(ok=True, error=None, duration_ms=0.0)
+
+ async def probe_brain_model(provider: str, model: str, **_kwargs: Any) -> Any:
+ recorder._note("model_probe", provider)
+ return provider_test.ProviderTestResult(provider, provider_test.OK)
+
+ async def probe_polish(family: Any, cfg: Any, **_kwargs: Any) -> Any:
+ recorder._note("polish_probe", family)
+ return SimpleNamespace(status="ok", detail="")
+
+ def instantiate(_self: Any, name: str, **_kwargs: Any) -> Any:
+ recorder._note("brain_instantiate", name)
+ raise AssertionError("a health check built a brain provider")
+
+ monkeypatch.setattr(provider_test, "run_provider_test", run_provider_test)
+ monkeypatch.setattr(provider_test, "_default_realtime_probe", realtime_probe)
+ monkeypatch.setattr(provider_test, "_default_make_tts", make_tts)
+ monkeypatch.setattr(provider_test, "_default_make_stt", make_stt)
+ monkeypatch.setattr(healthcheck.BrainHealthChecker, "probe", checker_probe)
+ monkeypatch.setattr(provider_routes, "_probe_brain_model", probe_brain_model)
+ monkeypatch.setattr(polish_probe, "probe_polish_family", probe_polish)
+ monkeypatch.setattr(
+ provider_registry.BrainProviderRegistry, "instantiate", instantiate
+ )
+ return self
+
+
+__all__ = ["ProviderCallRecorder"]
diff --git a/tests/integration/test_provider_routes.py b/tests/integration/test_provider_routes.py
index f0b039431b..cd2de289d8 100644
--- a/tests/integration/test_provider_routes.py
+++ b/tests/integration/test_provider_routes.py
@@ -131,8 +131,8 @@ def test_list_providers_returns_full_catalog(server_with_brain: WebServer, secre
class _FakeTestResult:
- """Minimal stand-in for provider_test.ProviderTestResult — the section-health
- route only reads ``.status`` and ``.detail``."""
+ """Minimal stand-in for provider_test.ProviderTestResult (``.status`` and
+ ``.detail`` are all a caller reads)."""
def __init__(self, status: str = "ok", detail: str = "") -> None:
self.status = status
@@ -141,9 +141,9 @@ class _FakeTestResult:
@pytest.fixture
def no_real_provider_test(monkeypatch: pytest.MonkeyPatch) -> None:
- """Stub the REAL connectivity call so section-health never hits the network
- (and can't pick up the maintainer's live keyring keys), keeping the test
- hermetic and fast."""
+ """Belt and braces: section-health no longer runs the connectivity test at
+ all, but if a regression made it do so again, this keeps the suite off the
+ network (and off the maintainer's live keyring keys)."""
from jarvis.brain import provider_test as _pt
async def _fake_run(spec: Any, cfg: Any, **kwargs: Any) -> _FakeTestResult: # noqa: ANN401
@@ -222,14 +222,44 @@ def test_section_health_caches_then_refresh_bypasses(
)
+def _slow_credential_check(
+ monkeypatch: pytest.MonkeyPatch,
+ *,
+ is_slow: Any, # noqa: ANN401
+ started: asyncio.Event,
+ cancelled: asyncio.Event,
+) -> None:
+ """Make the (local, free) credential check of one selection hang.
+
+ The rollup no longer probes anything, so the slowest step left is a
+ credential/login read; the cancellation machinery must still keep a slow
+ one for an OLD selection from blocking or labelling the new one.
+ """
+ from jarvis.ui.web import provider_routes
+
+ original = provider_routes._provider_credential_present_for_binary_async
+
+ async def _check(spec: Any, binary_path: str | None) -> bool: # noqa: ANN401
+ if is_slow(spec):
+ started.set()
+ try:
+ await asyncio.Future()
+ except asyncio.CancelledError:
+ cancelled.set()
+ raise
+ return await original(spec, binary_path)
+
+ monkeypatch.setattr(provider_routes, "_provider_credential_present_for_binary_async", _check)
+
+
@pytest.mark.asyncio
async def test_section_health_switch_cancels_old_provider_without_misattribution(
server_with_brain: WebServer,
secret_store: _InMemorySecretStore,
monkeypatch: pytest.MonkeyPatch,
) -> None:
- """A slow NVIDIA result must never block or label the new OpenRouter card."""
- from jarvis.brain import provider_test as provider_test_module
+ """A slow NVIDIA check must never block or label the new OpenRouter card."""
+ from jarvis.brain import provider_health_ledger as ledger
from jarvis.ui.web import provider_routes
secret_store.data.update(
@@ -238,21 +268,16 @@ async def test_section_health_switch_cancels_old_provider_without_misattribution
"openrouter_api_key": "sk-or-test",
}
)
+ ledger.get_ledger().record("openrouter", ledger.MODALITY_BRAIN, "ok")
server_with_brain.app.state.brain.active_provider = "nvidia"
nvidia_started = asyncio.Event()
nvidia_cancelled = asyncio.Event()
-
- async def _probe(spec: Any, cfg: Any, **kwargs: Any) -> _FakeTestResult: # noqa: ANN401
- if spec.id == "nvidia":
- nvidia_started.set()
- try:
- await asyncio.Future()
- except asyncio.CancelledError:
- nvidia_cancelled.set()
- raise
- return _FakeTestResult("ok", "")
-
- monkeypatch.setattr(provider_test_module, "run_provider_test", _probe)
+ _slow_credential_check(
+ monkeypatch,
+ is_slow=lambda spec: spec.id == "nvidia",
+ started=nvidia_started,
+ cancelled=nvidia_cancelled,
+ )
monkeypatch.setattr(
provider_routes,
"_jarvis_agent_section_health",
@@ -277,36 +302,33 @@ async def test_section_health_switch_cancels_old_provider_without_misattribution
@pytest.mark.asyncio
-async def test_section_health_model_switch_cancels_old_probe_for_same_provider(
+async def test_section_health_model_switch_cancels_old_check_for_same_provider(
server_with_brain: WebServer,
secret_store: _InMemorySecretStore,
monkeypatch: pytest.MonkeyPatch,
) -> None:
- """A timeout from an old model must not label a new model on the same card."""
- from jarvis.brain import provider_test as provider_test_module
+ """A check still running for an old model must not label the new model."""
+ from jarvis.brain import provider_health_ledger as ledger
from jarvis.core.config import BrainProviderConfig
from jarvis.ui.web import provider_routes
secret_store.data["openrouter_api_key"] = "sk-or-test"
+ ledger.get_ledger().record("openrouter", ledger.MODALITY_BRAIN, "ok")
server_with_brain.app.state.brain.active_provider = "openrouter"
server_with_brain.cfg.brain.providers["openrouter"] = BrainProviderConfig(
model="slow-model"
)
old_started = asyncio.Event()
old_cancelled = asyncio.Event()
-
- async def _probe(spec: Any, cfg: Any, **kwargs: Any) -> _FakeTestResult: # noqa: ANN401
- selected_model = cfg.brain.providers["openrouter"].model
- if spec.id == "openrouter" and selected_model == "slow-model":
- old_started.set()
- try:
- await asyncio.Future()
- except asyncio.CancelledError:
- old_cancelled.set()
- raise
- return _FakeTestResult("ok", "")
-
- monkeypatch.setattr(provider_test_module, "run_provider_test", _probe)
+ _slow_credential_check(
+ monkeypatch,
+ is_slow=lambda spec: (
+ spec.id == "openrouter"
+ and server_with_brain.cfg.brain.providers["openrouter"].model == "slow-model"
+ ),
+ started=old_started,
+ cancelled=old_cancelled,
+ )
monkeypatch.setattr(
provider_routes,
"_jarvis_agent_section_health",
@@ -329,40 +351,159 @@ async def test_section_health_model_switch_cancels_old_probe_for_same_provider(
assert old_response.sections["brain"].status == "ok"
-def test_section_health_computer_use_probes_the_tool_model_pin(
+def test_section_health_computer_use_reads_the_tool_model_outcome(
server_with_brain: WebServer,
secret_store: _InMemorySecretStore,
monkeypatch: pytest.MonkeyPatch,
) -> None:
- """The Tool Model tab must test the model that tier actually runs.
+ """The Tool Model tab is judged by what tool-model calls did, not brain turns.
- Live macOS fresh-install bug 2026-07-17: the computer-use section probed
- the provider's general brain model ("" on a fresh install), which collapsed
- into the plugin's hardcoded default — a retired id that 404'd, painting the
- Tool Model tab red although the runtime resolution was healthy (AP-23).
+ Its tier runs its own model pin, so a retired tool-model id (404) must turn
+ that tab red while the same provider's brain turns keep answering — and it
+ is learnt from the real call, never from a probe of the page's own.
"""
- from jarvis.brain import provider_test as provider_test_module
+ from jarvis.brain import provider_health_ledger as ledger
from jarvis.core.config import BrainProviderConfig, BrainTierConfig
+ from tests.fakes.fake_provider_calls import ProviderCallRecorder
+ recorder = ProviderCallRecorder().install(monkeypatch)
secret_store.data["gemini_api_key"] = "AIza-test"
+ server_with_brain.app.state.brain.active_provider = "gemini"
server_with_brain.cfg.brain.tool_model = BrainTierConfig(provider="gemini")
server_with_brain.cfg.brain.providers["gemini"] = BrainProviderConfig(
model="general-brain-model", tool_model="pinned-tool-model"
)
+ record = ledger.get_ledger()
+ record.record("gemini", ledger.MODALITY_BRAIN, "ok")
+ record.record("gemini", ledger.MODALITY_TOOL, "model_unavailable", model="pinned-tool-model")
- seen: dict[str, Any] = {}
+ with TestClient(server_with_brain.app) as client:
+ body = client.get("/api/providers/section-health").json()
+
+ assert body["sections"]["computer-use"]["subject_id"] == "gemini"
+ assert body["sections"]["computer-use"]["status"] == "error"
+ assert body["sections"]["computer-use"]["reason"] == "model_unavailable"
+ assert body["sections"]["brain"]["status"] == "ok"
+ assert recorder.calls == []
+
+
+# Every family a single-key install might hold, so every keyed tier is
+# "configured" — exactly the state in which the rollup used to send a probe.
+_ALL_FAMILY_KEYS = {
+ "openai_api_key": "sk-test",
+ "gemini_api_key": "AIza-test",
+ "openrouter_api_key": "sk-or-test",
+ "grok_api_key": "xai-test",
+ "anthropic_api_key": "sk-ant-test",
+ "nvidia_api_key": "nvapi-test",
+ "groq_api_key": "gsk-test",
+ "elevenlabs_api_key": "el-test",
+ "cartesia_api_key": "ca-test",
+ "inworld_api_key": "iw-test",
+}
+
+
+def test_section_health_and_readiness_send_nothing_to_any_provider(
+ server_with_brain: WebServer,
+ secret_store: _InMemorySecretStore,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """The rollup reads credentials and the passive record — zero provider calls.
+
+ Live 2026-09-30: one TTS synthesis and one billed GPT-Live session start per
+ app start / window reload, from this route alone. Mounting the sidebar,
+ the dock, the ready note and the API-Keys page, forced refresh included,
+ must not reach a provider any more.
+ """
+ from tests.fakes.fake_provider_calls import ProviderCallRecorder
+
+ recorder = ProviderCallRecorder().install(monkeypatch)
+ secret_store.data.update(_ALL_FAMILY_KEYS)
+
+ with TestClient(server_with_brain.app) as client:
+ first = client.get("/api/providers/section-health")
+ forced = client.get("/api/providers/section-health?refresh=true")
+ ready = client.get("/api/setup/readiness?refresh=true")
+
+ assert first.status_code == forced.status_code == ready.status_code == 200
+ assert recorder.calls == []
+ brain = forced.json()["sections"]["brain"]
+ # Set up, never used since: silent, and checked by its first real call.
+ assert brain["status"] == "unknown"
+ assert brain["reason"] == "unverified"
+
+
+def test_section_health_reports_the_last_real_outcome(
+ server_with_brain: WebServer,
+ secret_store: _InMemorySecretStore,
+) -> None:
+ """A real call's outcome shows up on the very next read — no refresh, no probe."""
+ from jarvis.brain import provider_health_ledger as ledger
+
+ secret_store.data["openai_api_key"] = "sk-test"
+ with TestClient(server_with_brain.app) as client:
+ before = client.get("/api/providers/section-health").json()
+ ledger.get_ledger().record("openai", ledger.MODALITY_BRAIN, "bad_key")
+ after = client.get("/api/providers/section-health").json()
+
+ assert before["sections"]["brain"]["status"] == "unknown"
+ assert after["cached"] is False
+ assert after["sections"]["brain"]["status"] == "error"
+ assert after["sections"]["brain"]["reason"] == "bad_key"
+ assert "last real call" in after["sections"]["brain"]["detail"]
- async def _probe(spec: Any, cfg: Any, **kwargs: Any) -> _FakeTestResult: # noqa: ANN401
- seen[spec.id] = kwargs.get("model")
- return _FakeTestResult("ok", "")
+
+def test_the_test_button_verdict_feeds_the_record(
+ server_with_brain: WebServer,
+ secret_store: _InMemorySecretStore,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """The user-clicked Test is the one real probe left, and its verdict is
+ what the dots show afterwards — without a second probe on the re-read."""
+ from jarvis.brain import provider_health_ledger as ledger
+ from jarvis.brain import provider_test as provider_test_module
+
+ secret_store.data["openai_api_key"] = "sk-test"
+ probes: list[str] = []
+
+ async def _probe(spec: Any, cfg: Any, **kwargs: Any) -> Any: # noqa: ANN401
+ probes.append(spec.id)
+ return provider_test_module.ProviderTestResult(spec.id, "no_credits", "HTTP 429")
monkeypatch.setattr(provider_test_module, "run_provider_test", _probe)
with TestClient(server_with_brain.app) as client:
- body = client.get("/api/providers/section-health?refresh=true").json()
+ verdict = client.post("/api/providers/openai/test").json()
+ body = client.get("/api/providers/section-health").json()
- assert body["sections"]["computer-use"]["subject_id"] == "gemini"
- assert seen["gemini"] == "pinned-tool-model"
+ assert verdict["status"] == "no_credits"
+ assert probes == ["openai"]
+ outcome = ledger.get_ledger().get("openai", ledger.MODALITY_BRAIN)
+ assert outcome is not None and outcome.source == ledger.SOURCE_TEST
+ assert body["sections"]["brain"]["status"] == "error"
+ assert body["sections"]["brain"]["reason"] == "no_credits"
+
+
+def test_saving_a_key_forgets_what_the_old_key_did(
+ server_with_brain: WebServer,
+ secret_store: _InMemorySecretStore,
+) -> None:
+ """A rejected key's red dot must not outlive the key; the new one starts clean."""
+ from jarvis.brain import provider_health_ledger as ledger
+
+ record = ledger.get_ledger()
+ record.record("openai", ledger.MODALITY_BRAIN, "bad_key")
+ record.record("openai-live", ledger.MODALITY_REALTIME, "bad_key")
+ record.record("elevenlabs", ledger.MODALITY_TTS, "no_credits")
+
+ with TestClient(server_with_brain.app) as client:
+ resp = client.post("/api/secrets/openai_api_key", json={"value": "sk-new"})
+
+ assert resp.status_code == 200
+ assert record.get("openai", ledger.MODALITY_BRAIN) is None
+ assert record.get("openai-live", ledger.MODALITY_REALTIME) is None
+ # An unrelated provider keeps its history.
+ assert record.get("elevenlabs", ledger.MODALITY_TTS) is not None
def test_list_providers_exposes_credential_help_and_billing(
diff --git a/tests/integration/test_starter_plan_routes.py b/tests/integration/test_starter_plan_routes.py
index 7d1d1afeaf..090eb600cd 100644
--- a/tests/integration/test_starter_plan_routes.py
+++ b/tests/integration/test_starter_plan_routes.py
@@ -103,6 +103,41 @@ def test_readiness_is_false_when_a_required_section_is_not_ok(
assert body["sections"]["subagents"]["status"] == "needs_setup"
+def test_readiness_counts_a_set_up_but_unused_section_as_ready(
+ server, state_path, monkeypatch
+) -> None:
+ """The rollup never probes, so a freshly keyed tier is ``unknown`` /
+ ``unverified`` until its first real call. That is set up — the note must
+ not wait for every tier to have been used once. A KNOWN failure or any
+ other unknown still holds it back."""
+ server.cfg.voice.mode = "realtime"
+ unverified = {"status": "unknown", "reason": provider_routes.UNVERIFIED_REASON, "detail": ""}
+
+ async def fresh_install(request, refresh=False):
+ snap = _all_ok(("subagents",))
+ snap.sections["realtime"] = dict(unverified)
+ snap.sections["computer-use"] = dict(unverified)
+ return snap
+
+ async def one_known_failure(request, refresh=False):
+ snap = await fresh_install(request)
+ snap.sections["computer-use"] = {"status": "error", "reason": "bad_key", "detail": ""}
+ return snap
+
+ async def unknown_for_another_reason(request, refresh=False):
+ snap = await fresh_install(request)
+ snap.sections["realtime"] = {"status": "unknown", "reason": "error", "detail": ""}
+ return snap
+
+ with TestClient(server.app) as client:
+ monkeypatch.setattr(provider_routes, "section_health", fresh_install)
+ assert client.get("/api/setup/readiness").json()["ready"] is True
+ monkeypatch.setattr(provider_routes, "section_health", one_known_failure)
+ assert client.get("/api/setup/readiness").json()["ready"] is False
+ monkeypatch.setattr(provider_routes, "section_health", unknown_for_another_reason)
+ assert client.get("/api/setup/readiness").json()["ready"] is False
+
+
def test_readiness_fails_open_when_health_probe_breaks(server, state_path, monkeypatch) -> None:
async def boom(request, refresh=False):
raise RuntimeError("probe exploded")
diff --git a/tests/unit/agent_chat/test_agent_chat_service.py b/tests/unit/agent_chat/test_agent_chat_service.py
index 717967e348..baa250f2d6 100644
--- a/tests/unit/agent_chat/test_agent_chat_service.py
+++ b/tests/unit/agent_chat/test_agent_chat_service.py
@@ -228,8 +228,9 @@ def _app(tmp_path: Path) -> FastAPI:
def test_provider_health_reports_each_row_and_caches_the_sweep(tmp_path: Path, monkeypatch):
"""The picker's live state: which connected seats actually answer.
- The sweep costs one real request per provider, so it must be cached and
- must never lose the whole answer to one slow or raising provider.
+ The sweep reads key presence, CLI logins and the passive health record;
+ the login reads are not free, so it is cached, and it must never lose the
+ whole answer to one slow or raising row.
"""
import jarvis.ui.web.agent_chat_routes as routes
@@ -240,7 +241,7 @@ def test_provider_health_reports_each_row_and_caches_the_sweep(tmp_path: Path, m
cli_calls.append(runner)
return "ok", "ok", f"{runner}: signed in"
- async def _fake(cfg, provider_id, *, probe=True):
+ async def _fake(cfg, provider_id):
calls.append(provider_id)
table = {
"claude-api": ("error", "bad_key", "Claude (API-Key): 401"),
@@ -286,6 +287,40 @@ def test_provider_health_reports_each_row_and_caches_the_sweep(tmp_path: Path, m
assert len(calls) > swept[0] and len(cli_calls) > swept[1]
+def test_provider_health_sweep_sends_nothing_to_any_provider(tmp_path: Path, monkeypatch):
+ """Every open of the chat used to spend one paid completion per keyed
+ provider (live 2026-09-30). The composer now reads the passive record of
+ real calls: zero provider requests, and a new real outcome drops the
+ cached sweep so the next open shows it."""
+ import jarvis.ui.web.agent_chat_routes as routes
+ from jarvis.brain import provider_health_ledger as ledger
+ from jarvis.ui.web import provider_routes
+ from tests.fakes.fake_provider_calls import ProviderCallRecorder
+
+ recorder = ProviderCallRecorder().install(monkeypatch)
+ monkeypatch.setattr(provider_routes, "_is_credential_present", lambda *args: True)
+ monkeypatch.setattr(
+ routes, "_cli_login_snapshot", lambda runner: ("ok", "ok", f"{runner}: signed in")
+ )
+ monkeypatch.setattr("jarvis.agent_chat.service._claude_cli_installed", lambda: False)
+ monkeypatch.setattr(routes, "_health_cache", {})
+
+ with TestClient(_app(tmp_path)) as client:
+ first = client.get("/api/agent-chat/provider-health?surface=jarvis").json()
+ rows = {r["provider"]: r for r in first["providers"]}
+ assert rows["grok"]["status"] == "unknown" # keyed, never used: no dot
+ assert rows["grok"]["reason"] == provider_routes.UNVERIFIED_REASON
+
+ ledger.get_ledger().record("grok", ledger.MODALITY_BRAIN, "no_credits")
+ second = client.get("/api/agent-chat/provider-health?surface=jarvis").json()
+
+ assert second["cached"] is False
+ rows = {r["provider"]: r for r in second["providers"]}
+ assert rows["grok"]["status"] == "error"
+ assert rows["grok"]["reason"] == "no_credits"
+ assert recorder.calls == []
+
+
class _CliStatus:
def __init__(self, connected: bool, mode: str, message: str = "") -> None:
self.connected = connected
@@ -357,7 +392,7 @@ def test_cli_surface_health_does_not_probe_the_api_key(tmp_path: Path, monkeypat
api_calls: list[str] = []
- async def _fake_api(cfg, provider_id, *, probe=True):
+ async def _fake_api(cfg, provider_id):
api_calls.append(provider_id)
return SimpleNamespace(status="error", reason="bad_key", detail=f"{provider_id}: 401")
@@ -393,7 +428,7 @@ def test_jarvis_surface_probes_the_claude_api_key_when_cli_is_uninstalled(
api_calls: list[str] = []
cli_calls: list[str] = []
- async def _fake_api(cfg, provider_id, *, probe=True):
+ async def _fake_api(cfg, provider_id):
api_calls.append(provider_id)
return SimpleNamespace(status="error", reason="bad_key", detail="401")
diff --git a/tests/unit/brain/test_section_health.py b/tests/unit/brain/test_section_health.py
index 7c4325fbae..c707a8bbcf 100644
--- a/tests/unit/brain/test_section_health.py
+++ b/tests/unit/brain/test_section_health.py
@@ -83,15 +83,17 @@ async def test_every_catalog_provider_health_is_bound_to_its_exact_id(
"""The generic guard covers every Brain, TTS, STT, Realtime and wording card.
What is under test is the BINDING (a tier's health names the exact provider
- it came from), so every way a card can be judged is stubbed healthy here:
- the shared probe, the on-device readiness check that answers for keyless
- cards, and the wording probe that owns the dictation tier. A card that
- reached a real provider instead would make this a connectivity test of the
- machine it runs on.
+ it came from), so every way a card can be judged reads healthy here: a
+ recorded real-call success for its modality, the on-device readiness check
+ that answers for keyless cards, and credential presence for the wording
+ tier. No card may reach a provider on the way — the recorder proves it.
"""
from jarvis import codex_app_server
- from jarvis.dictation import polish_probe
+ from jarvis.brain import provider_health_ledger as ledger
from jarvis.ui.web import provider_routes
+ from tests.fakes.fake_provider_calls import ProviderCallRecorder
+
+ recorder = ProviderCallRecorder().install(monkeypatch)
monkeypatch.setattr(provider_routes, "_is_credential_present", lambda *args: True)
@@ -123,19 +125,17 @@ async def test_every_catalog_provider_health_is_bound_to_its_exact_id(
lambda _spec: {"ready": True, "detail": "installed"},
)
- async def _probe(selected, cfg, **kwargs):
- return SimpleNamespace(status="ok", detail="")
-
- async def _polish_probe(family, cfg, **kwargs):
- return SimpleNamespace(status="ok", detail="")
+ modality = ledger.modality_for_tier(spec.tier)
+ if modality is not None:
+ ledger.get_ledger().record(spec.id, modality, provider_test.OK)
- monkeypatch.setattr(provider_test, "run_provider_test", _probe)
- monkeypatch.setattr(polish_probe, "probe_polish_family", _polish_probe)
-
- result = await provider_routes._tier_section_health(SimpleNamespace(), spec)
+ result = await provider_routes._tier_section_health(
+ SimpleNamespace(), spec, modality=modality
+ )
assert result.status == sh.OK
assert result.subject_id == spec.id
+ assert recorder.calls == []
def test_health_fingerprint_covers_every_model_selection_surface() -> None:
@@ -337,80 +337,132 @@ def _gpt_live_config(mode: str = "realtime") -> SimpleNamespace:
)
-async def _brain_probe_flag(monkeypatch, cfg, brain_subject: str) -> bool:
- """Run the rollup with every check stubbed; return the brain ``probe`` flag."""
+async def _section_modalities(monkeypatch, cfg) -> dict[str, str | None]:
+ """Run the rollup with every check stubbed; return each tier's modality."""
from jarvis.ui.web import provider_routes as pr
- seen: dict[str, bool] = {}
+ seen: dict[str, str | None] = {}
- async def _tier(cfg, spec, *, probe=True, **kwargs):
- if spec is not None and spec.id == brain_subject:
- seen["brain"] = probe
- return pr.SectionHealth(status=sh.OK, reason="ok")
-
- async def _other(*args, **kwargs):
+ async def _tier(cfg, spec, *, modality=None, **kwargs):
+ if spec is not None:
+ seen[spec.id] = modality
return pr.SectionHealth(status=sh.OK, reason="ok")
def _sync_other(*args, **kwargs):
return pr.SectionHealth(status=sh.OK, reason="ok")
monkeypatch.setattr(pr, "_tier_section_health", _tier)
- monkeypatch.setattr(pr, "_realtime_section_health", _other)
- monkeypatch.setattr(pr, "_dictation_section_health", _other)
monkeypatch.setattr(pr, "_jarvis_agent_section_health", _sync_other)
monkeypatch.setattr(pr, "_advanced_section_health", _sync_other)
monkeypatch.setattr(pr, "_local_models_section_health", _sync_other)
monkeypatch.setattr(pr, "_codex_binary_path", lambda request: None)
monkeypatch.setattr(pr, "_polish_enabled", lambda cfg: False)
subjects = {key: None for key in pr._SECTION_HEALTH_KEYS}
- subjects["brain"] = brain_subject
+ subjects.update(
+ {
+ "brain": "openai",
+ "computer-use": "grok",
+ "tts": "elevenlabs",
+ "stt": "groq-api",
+ "realtime": "openai-live",
+ }
+ )
await pr._compute_section_health(SimpleNamespace(), cfg, subjects)
- return seen["brain"]
+ return seen
@pytest.mark.asyncio
-async def test_brain_on_the_voice_key_is_not_probed_twice(monkeypatch) -> None:
- """The realtime section already spends one call on the GPT-Live key; the
- hidden Brain tab must not spend a second one on the same key (2026-09-29)."""
- assert await _brain_probe_flag(monkeypatch, _gpt_live_config(), "openai") is False
+async def test_every_tier_reads_its_own_real_call_outcomes(monkeypatch) -> None:
+ """Each tab is judged by the record of ITS calls: brain turns for Brain,
+ tool-model steps for Tool Model, spoken sentences, transcriptions and
+ realtime handshakes — never by a probe of the page's own (2026-09-30)."""
+ from jarvis.brain import provider_health_ledger as ledger
+
+ seen = await _section_modalities(monkeypatch, _gpt_live_config())
+
+ assert seen == {
+ "openai": ledger.MODALITY_BRAIN,
+ "grok": ledger.MODALITY_TOOL,
+ "elevenlabs": ledger.MODALITY_TTS,
+ "groq-api": ledger.MODALITY_STT,
+ "openai-live": ledger.MODALITY_REALTIME,
+ }
@pytest.mark.asyncio
-async def test_brain_off_the_voice_key_is_still_probed(monkeypatch) -> None:
- assert await _brain_probe_flag(monkeypatch, _gpt_live_config(), "grok") is True
+async def test_a_keyed_provider_never_used_is_silent_not_red(monkeypatch) -> None:
+ """Credential present, no real call since: ``unknown`` — no dot, no probe."""
+ from jarvis.brain import provider_health_ledger as ledger
+ from jarvis.ui.web import provider_routes as pr
+ from tests.fakes.fake_provider_calls import ProviderCallRecorder
+
+ recorder = ProviderCallRecorder().install(monkeypatch)
+ monkeypatch.setattr(pr, "_is_credential_present", lambda *args: True)
+
+ health = await pr._tier_section_health(
+ None, pr.get_spec("elevenlabs"), modality=ledger.MODALITY_TTS
+ )
+
+ assert health.status == sh.UNKNOWN
+ assert health.reason == pr.UNVERIFIED_REASON
+ assert recorder.calls == []
@pytest.mark.asyncio
-async def test_pipeline_voice_probes_an_openai_brain(monkeypatch) -> None:
- cfg = _gpt_live_config(mode="pipeline")
+async def test_a_login_based_provider_is_ok_on_its_login(monkeypatch) -> None:
+ """A signed-in CLI login is the whole verdict (the Test button gives no more)."""
+ from jarvis.brain import provider_health_ledger as ledger
+ from jarvis.ui.web import provider_routes as pr
- assert await _brain_probe_flag(monkeypatch, cfg, "openai") is True
+ monkeypatch.setattr(pr, "_is_credential_present", lambda *args: True)
+
+ health = await pr._tier_section_health(
+ None, pr.get_spec("codex"), modality=ledger.MODALITY_BRAIN
+ )
+
+ assert health.status == sh.OK
+ assert health.reason == "connected"
def test_passive_health_polls_reuse_the_cache_for_minutes() -> None:
- """Sidebar and dock poll this in every window and every build reloads every
- window; a sub-minute TTL turned that into a paid probe per minute."""
+ """Sidebar and dock read this in every window and every build reloads every
+ window; the credential/login reads behind it are cached for minutes."""
from jarvis.ui.web import provider_routes as pr
assert pr._SECTION_HEALTH_TTL_S >= 15 * 60
-async def test_composer_health_does_not_spend_the_voice_key(monkeypatch) -> None:
- """The agent-chat composer sweeps every API row; the GPT-Live key must be
- reported from configuration, not probed on every sweep (2026-09-29)."""
+def test_a_new_real_call_outcome_supersedes_the_cached_rollup() -> None:
+ from jarvis.brain import provider_health_ledger as ledger
from jarvis.ui.web import provider_routes as pr
- probes: dict[str, bool] = {}
+ cfg = SimpleNamespace(brain=SimpleNamespace(providers={}))
+ request = SimpleNamespace(
+ app=SimpleNamespace(state=SimpleNamespace(telephony_manager=None))
+ )
+ subjects = {key: None for key in pr._SECTION_HEALTH_KEYS}
+ before = pr._section_health_fingerprint(request, cfg, subjects)
+ ledger.get_ledger().record("openai", ledger.MODALITY_BRAIN, provider_test.BAD_KEY)
- async def _tier(cfg, spec, *, probe=True, **kwargs):
- probes[spec.id] = probe
- return pr.SectionHealth(status=sh.OK, reason="ok", subject_id=spec.id)
+ assert pr._section_health_fingerprint(request, cfg, subjects) != before
- monkeypatch.setattr(pr, "_tier_section_health", _tier)
+
+async def test_composer_health_reads_the_record_and_sends_nothing(monkeypatch) -> None:
+ """The agent-chat composer used to spend one paid completion per keyed
+ provider on every open. It now reads the same record the tabs read."""
+ from jarvis.brain import provider_health_ledger as ledger
+ from jarvis.ui.web import provider_routes as pr
+ from tests.fakes.fake_provider_calls import ProviderCallRecorder
+
+ recorder = ProviderCallRecorder().install(monkeypatch)
+ monkeypatch.setattr(pr, "_is_credential_present", lambda *args: True)
+ ledger.get_ledger().record("grok", ledger.MODALITY_BRAIN, provider_test.NO_CREDITS)
cfg = _gpt_live_config()
- await pr.provider_health(cfg, "openai", probe=True)
- await pr.provider_health(cfg, "grok", probe=True)
+ grok = await pr.provider_health(cfg, "grok")
+ openai = await pr.provider_health(cfg, "openai")
- assert probes == {"openai": False, "grok": True}
+ assert (grok.status, grok.reason) == (sh.ERROR, provider_test.NO_CREDITS)
+ assert (openai.status, openai.reason) == (sh.UNKNOWN, pr.UNVERIFIED_REASON)
+ assert recorder.calls == []
diff --git a/tests/unit/ui/test_dictation_provider_tier.py b/tests/unit/ui/test_dictation_provider_tier.py
index d7ce7bead2..d161f79e19 100644
--- a/tests/unit/ui/test_dictation_provider_tier.py
+++ b/tests/unit/ui/test_dictation_provider_tier.py
@@ -177,8 +177,8 @@ async def test_switching_the_pass_off_draws_no_dot() -> None:
async def test_a_present_key_reports_ready_without_a_live_call(monkeypatch) -> None:
- """``probe=False``: the credential here is always a key another tier already
- tests, so page-open must not fire a second network round-trip."""
+ """Credential presence decides: the key here is always one another tier
+ already uses, and page-open must never fire a network round-trip."""
monkeypatch.setattr(pr, "_is_credential_present", lambda spec, bp=None: True)
async def _explode(*_args, **_kwargs):
@@ -208,23 +208,17 @@ async def test_a_required_tier_still_nags(monkeypatch) -> None:
async def test_an_optional_tier_still_reports_a_failing_key(monkeypatch) -> None:
"""Nag suppression covers "not set up", never "set up and broken"."""
- monkeypatch.setattr(pr, "_is_credential_present", lambda spec, bp=None: True)
-
- async def _bad_key(spec, cfg, **_kwargs):
- return SimpleNamespace(status="bad_key", detail="rejected")
-
- # A wording card is judged by the dictation layer's own probe, never by the
- # STT branch of the shared provider test — the tier has no recognizer behind
- # it, and probing one reported the user's speech engine under this card's
- # name. Both seams are stubbed so the test pins the RULE (an optional tier
- # still goes red on a failing key) rather than which module answered.
- from jarvis.dictation import polish_probe
+ from jarvis.brain import provider_health_ledger as ledger
- monkeypatch.setattr(pr._provider_test, "run_provider_test", _bad_key)
- monkeypatch.setattr(polish_probe, "probe_polish_family", _bad_key)
+ monkeypatch.setattr(pr, "_is_credential_present", lambda spec, bp=None: True)
spec = pr.get_spec(dictation_spec_id("groq"))
+ # A real call on this card's key was rejected: the optional flag must not
+ # hide that. (The record, not a probe, carries the evidence.)
+ ledger.get_ledger().record(spec.id, ledger.MODALITY_BRAIN, "bad_key")
- health = await pr._tier_section_health(None, spec, probe=True)
+ health = await pr._tier_section_health(
+ None, spec, modality=ledger.MODALITY_BRAIN, optional=True
+ )
assert health.status != _section_health.OK
assert health.reason == "bad_key"
diff --git a/tests/unit/web/test_realtime_provider_category.py b/tests/unit/web/test_realtime_provider_category.py
index 1f2adbb27c..51ab94db43 100644
--- a/tests/unit/web/test_realtime_provider_category.py
+++ b/tests/unit/web/test_realtime_provider_category.py
@@ -218,6 +218,8 @@ def test_section_health_includes_realtime_key(monkeypatch):
return SimpleNamespace(status="ok", detail="")
+ from jarvis.brain import provider_health_ledger as ledger
+
monkeypatch.setattr(_pt, "run_provider_test", _fake_run)
monkeypatch.setattr(cfg_mod, "get_secret", _only_openai_key)
client = TestClient(_app())
@@ -225,6 +227,15 @@ def test_section_health_includes_realtime_key(monkeypatch):
assert resp.status_code == 200
sections = resp.json()["sections"]
assert "realtime" in sections
+ # Key present, no call yet: silent — the tab never opens a (billed)
+ # session of its own to find out.
+ assert sections["realtime"]["status"] == "unknown"
+ assert sections["realtime"]["reason"] == "unverified"
+
+ # The first real handshake is the evidence.
+ subject = sections["realtime"]["subject_id"]
+ ledger.get_ledger().record(subject, ledger.MODALITY_REALTIME, "ok")
+ sections = client.get("/api/providers/section-health").json()["sections"]
assert sections["realtime"]["status"] == "ok"