The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
"""A curated fallback served because the live catalog fetch failed must never be pinned in the
disk cache as if it were the account's real catalog (#107391): a transient Copilot outage wrote
the 17-model static list over the account's 10 live models with a fresh 1h TTL, and every picker
surface served the wrong list until the TTL lapsed.
"""
from __future__ import annotations
import time
from unittest.mock import patch
import pytest
import hermes_cli.models as mod
@pytest.fixture(autouse=True)
def _reset_swr_state():
with mod._swr_refresh_lock:
mod._swr_refresh_inflight.clear()
yield
with mod._swr_refresh_lock:
mod._swr_refresh_inflight.clear()
def _row(models, age_seconds, **extra):
return {"fp": "fp", "at": time.time() - age_seconds, "models": list(models), **extra}
LIVE = ["claude-opus-5", "kimi-k3", "gpt-5.6-sol"]
STATIC = ["gpt-5.4", "gpt-4o", "claude-sonnet-4.6"]
def test_fallback_does_not_overwrite_the_same_credentials_live_row():
# 2h-old live row (past the 1h TTL, inside the SWR window) + a live fetch that degrades to the
# curated list: the account's real catalog must survive, in the cache AND in the return value.
cache = {"deepinfra": _row(LIVE, age_seconds=7200)}
with patch.object(mod, "_load_provider_models_cache", return_value=cache), \
patch.object(mod, "_credential_fingerprint", return_value="fp"), \
patch.object(mod, "_save_provider_models_cache") as save, \
patch.object(mod, "provider_model_ids", return_value=mod.CuratedFallbackModels(STATIC)):
out = mod.cached_provider_model_ids("deepinfra", force_refresh=True)
assert out == LIVE
assert cache["deepinfra"]["models"] == LIVE
save.assert_not_called()
def test_fallback_row_is_recorded_as_fallback_and_never_served_stale():
# Cold cache: the curated list is served (the picker must not be empty) but recorded as a
# fallback row, which the SWR stale window must not resurrect — a stale fallback re-probes.
cache: dict = {}
with patch.object(mod, "_load_provider_models_cache", return_value=cache), \
patch.object(mod, "_credential_fingerprint", return_value="fp"), \
patch.object(mod, "_save_provider_models_cache"), \
patch.object(mod, "provider_model_ids", return_value=mod.CuratedFallbackModels(STATIC)):
assert mod.cached_provider_model_ids("deepinfra") == STATIC
assert cache["deepinfra"]["fallback"] is True
# Same row, now past the fallback TTL: the live fetch runs instead of the stale-serve path,
# and a real catalog replaces the fallback row.
cache["deepinfra"]["at"] = time.time() - mod._PROVIDER_MODELS_FALLBACK_TTL - 1
with patch.object(mod, "_load_provider_models_cache", return_value=cache), \
patch.object(mod, "_credential_fingerprint", return_value="fp"), \
patch.object(mod, "_save_provider_models_cache"), \
patch.object(mod, "_spawn_swr_refresh") as spawn, \
patch.object(mod, "provider_model_ids", return_value=list(LIVE)) as live:
assert mod.cached_provider_model_ids("deepinfra") == LIVE
live.assert_called_once()
spawn.assert_not_called()
assert cache["deepinfra"]["models"] == LIVE
assert "fallback" not in cache["deepinfra"]
def test_codex_cold_fallback_retries_after_short_ttl(monkeypatch, tmp_path):
from hermes_cli import auth, codex_models
monkeypatch.setenv("CODEX_HOME", str(tmp_path))
monkeypatch.setattr(auth, "resolve_codex_runtime_credentials", lambda **_: {"api_key": "test-token"})
monkeypatch.setattr(auth, "_codex_access_token_is_expiring", lambda *_: False)
responses = iter([[], ["gpt-6-astra"]])
monkeypatch.setattr(codex_models, "_fetch_models_from_api", lambda *_args, **_kw: next(responses))
cache = {}
with patch.object(mod, "_load_provider_models_cache", return_value=cache), \
patch.object(mod, "_credential_fingerprint", return_value="fp"), \
patch.object(mod, "_save_provider_models_cache"), \
patch.object(mod, "_spawn_swr_refresh") as spawn:
mod.cached_provider_model_ids("openai-codex")
assert cache["openai-codex"]["fallback"] is True
cache["openai-codex"]["at"] = time.time() - mod._PROVIDER_MODELS_FALLBACK_TTL - 1
assert "gpt-6-astra" in mod.cached_provider_model_ids("openai-codex")
assert "fallback" not in cache["openai-codex"]
spawn.assert_not_called()
def test_anthropic_catalog_marks_the_curated_list_as_fallback_on_a_failed_live_fetch():
# _anthropic_catalog used to `return curated` (a plain list) on a failed live fetch, so a
# transient outage on a proxy base_url cached the generic curated list over the account's/
# proxy's real catalog for a full hour (same class of bug as #107391, just missed by 3fb8b3f6).
with patch.object(mod, "_get_model_config_dict", return_value={}), \
patch.object(mod, "_fetch_anthropic_models", return_value=None):
rows = mod.provider_model_ids("anthropic")
assert isinstance(rows, mod.CuratedFallbackModels)
assert rows == list(mod._PROVIDER_MODELS["anthropic"])