Skip to content

The agent that grows with you

test(models): drive catalog backoff expiry through the clock

Teknium7354412817a9
diff --git a/hermes_cli/model_catalog.py b/hermes_cli/model_catalog.py
index 53338aae6a1..0824df716e0 100644
--- a/hermes_cli/model_catalog.py
+++ b/hermes_cli/model_catalog.py
@@ -48,19 +48,33 @@ _HERMES_USER_AGENT = f"hermes-cli/{get_version_info().base_version}"
_catalog_cache: dict[str, Any] | None = None
_catalog_cache_source_mtime: float = 0.0
_catalog_cache_source_path: str = ""
-# ``monotonic()`` deadline before which a failed fetch key is not retried (see FETCH_RETRY_DELAY_SECONDS).
-_fetch_retry_after: dict[str, float] = {}
+# ``monotonic()`` deadline before which a failed fetch is not retried (see FETCH_RETRY_DELAY_SECONDS),
+# keyed by (profile home key, URL): one gateway process serves many profiles.
+_fetch_retry_after: dict[tuple[str, str], float] = {}
-def _backing_off(key: str) -> bool:
- return time.monotonic() < _fetch_retry_after.get(key, 0.0)
+def _backoff_key(url: str) -> tuple[str, str]:
+ from hermes_constants import hermes_home_key
+ return (hermes_home_key(), url)
-def _note_fetch_result(key: str, ok: bool) -> None:
+def _backing_off(url: str) -> bool:
+ return time.monotonic() < _fetch_retry_after.get(_backoff_key(url), 0.0)
+
+
+def _note_fetch_result(url: str, ok: bool) -> None:
if ok:
- _fetch_retry_after.pop(key, None)
+ _fetch_retry_after.pop(_backoff_key(url), None)
else:
- _fetch_retry_after[key] = time.monotonic() + FETCH_RETRY_DELAY_SECONDS
+ _fetch_retry_after[_backoff_key(url)] = time.monotonic() + FETCH_RETRY_DELAY_SECONDS
+
+
+def clear_fetch_backoff() -> None:
+ """Forget the active profile's fetch failures so an explicit refresh retries every URL."""
+ from hermes_constants import hermes_home_key
+ home = hermes_home_key()
+ for key in [k for k in _fetch_retry_after if k[0] == home]:
+ del _fetch_retry_after[key]
def _load_catalog_config() -> dict[str, Any]:
@@ -249,12 +263,10 @@ def get_catalog(*, force_refresh: bool = False) -> dict[str, Any]:
_spawn_catalog_swr_refresh(cfg["url"])
return _remember(disk_data, disk_mtime)
- # Keyed per cache file (profile) and URL, like the in-process copy above.
- backoff_key = f"{_cache_path()}|{cfg['url']}"
- if not force_refresh and disk_data is None and _backing_off(backoff_key):
+ if not force_refresh and disk_data is None and _backing_off(cfg["url"]):
return {}
fetched = _fetch_manifest_with_fallback(cfg["url"], DEFAULT_FETCH_TIMEOUT)
- _note_fetch_result(backoff_key, fetched is not None)
+ _note_fetch_result(cfg["url"], fetched is not None)
if fetched is not None:
_write_disk_cache(fetched)
new_disk_data, new_mtime = _read_disk_cache()
@@ -277,6 +289,7 @@ def refresh_catalogs() -> bool:
Blocking; run it off the event loop."""
if not _load_catalog_config()["enabled"]:
return False
+ clear_fetch_backoff() # provider overrides are re-asked too, not only the forced manifest
catalog = get_catalog(force_refresh=True)
try:
from hermes_cli.models import fetch_nous_recommended_models, fetch_openrouter_models
diff --git a/hermes_cli/models.py b/hermes_cli/models.py
index 6b194375bab..4290ed48552 100644
--- a/hermes_cli/models.py
+++ b/hermes_cli/models.py
@@ -2159,14 +2159,14 @@ def clear_provider_models_cache(provider: Optional[str] = None) -> None:
"""Drop one provider's cache entry, or wipe the whole cache (``provider=None``). Used by
``/model --refresh`` and ``hermes model --refresh``."""
try:
- # Native Ollama tags are keyed by root URL, not provider slug — a targeted refresh can't
- # identify the root from the name alone, so clear this small in-process cache every time.
+ from hermes_cli.model_catalog import clear_fetch_backoff
+ clear_fetch_backoff() # a recovered catalog is fetched now, not after the offline backoff
+ # Native Ollama tags are keyed by root URL, not provider slug: clear them on every refresh.
_OLLAMA_LOCAL_MODELS_CACHE.clear()
_OLLAMA_LOCAL_PROBE_FAILURE_CACHE.clear()
_OLLAMA_LOCAL_PROBE_REACHABLE.clear()
- # A fresh copilot-acp CLI login must be visible to the next /model switch (this helper is
- # what ``--refresh`` runs): don't let the 5-min session memo (or its failure memo) serve
- # a stale signed-out probe past an explicit refresh.
+ # A fresh copilot-acp CLI login must show on the next /model switch: ``--refresh`` drops the
+ # 5-min session memo (and its failure memo) so it never serves a stale signed-out probe.
global _copilot_acp_session_memo
_copilot_acp_session_memo = None
from hermes_cli.models_cache_lock import write_under_cache_lock
diff --git a/tests/hermes_cli/test_model_catalog.py b/tests/hermes_cli/test_model_catalog.py
index 5bc8c37a5c0..74b00cdd492 100644
--- a/tests/hermes_cli/test_model_catalog.py
+++ b/tests/hermes_cli/test_model_catalog.py
@@ -134,6 +134,7 @@ class TestFetchFailure:
until the retry window ends; an explicit refresh still tries the network."""
from hermes_cli import model_catalog
calls: list[str] = []
+ failed_at = time.monotonic()
def offline(url, timeout):
calls.append(url) # returns None: the fetch failed
@@ -148,10 +149,9 @@ class TestFetchFailure:
assert model_catalog.get_catalog(force_refresh=True) == {}
assert len(calls) == 2 * per_attempt
- # Window over (zero-length): the next call fetches again and serves the manifest.
- monkeypatch.setattr(model_catalog, "FETCH_RETRY_DELAY_SECONDS", 0.0)
- with patch.object(model_catalog, "_fetch_manifest", side_effect=offline):
- assert model_catalog.get_catalog(force_refresh=True) == {}
+ # The clock passes the last failure's window: a plain call fetches again and serves it.
+ later = failed_at + model_catalog.FETCH_RETRY_DELAY_SECONDS + 60.0
+ monkeypatch.setattr(model_catalog.time, "monotonic", lambda: later)
with patch.object(model_catalog, "_fetch_manifest", return_value=_valid_manifest()):
assert model_catalog.get_catalog() == _valid_manifest()