Skip to content

The agent that grows with you

5.0 KB
Raw Blame
"""A curated fallback served because the live catalog fetch failed must never be pinned in the
disk cache as if it were the account's real catalog (#107391): a transient Copilot outage wrote
the 17-model static list over the account's 10 live models with a fresh 1h TTL, and every picker
surface served the wrong list until the TTL lapsed.
"""

from __future__ import annotations

import time
from unittest.mock import patch

import pytest

import hermes_cli.models as mod


@pytest.fixture(autouse=True)
def _reset_swr_state():
    with mod._swr_refresh_lock:
        mod._swr_refresh_inflight.clear()
    yield
    with mod._swr_refresh_lock:
        mod._swr_refresh_inflight.clear()


def _row(models, age_seconds, **extra):
    return {"fp": "fp", "at": time.time() - age_seconds, "models": list(models), **extra}


LIVE = ["claude-opus-5", "kimi-k3", "gpt-5.6-sol"]
STATIC = ["gpt-5.4", "gpt-4o", "claude-sonnet-4.6"]


def test_fallback_does_not_overwrite_the_same_credentials_live_row():
    # 2h-old live row (past the 1h TTL, inside the SWR window) + a live fetch that degrades to the
    # curated list: the account's real catalog must survive, in the cache AND in the return value.
    cache = {"deepinfra": _row(LIVE, age_seconds=7200)}
    with patch.object(mod, "_load_provider_models_cache", return_value=cache), \
         patch.object(mod, "_credential_fingerprint", return_value="fp"), \
         patch.object(mod, "_save_provider_models_cache") as save, \
         patch.object(mod, "provider_model_ids", return_value=mod.CuratedFallbackModels(STATIC)):
        out = mod.cached_provider_model_ids("deepinfra", force_refresh=True)

    assert out == LIVE
    assert cache["deepinfra"]["models"] == LIVE
    save.assert_not_called()


def test_fallback_row_is_recorded_as_fallback_and_never_served_stale():
    # Cold cache: the curated list is served (the picker must not be empty) but recorded as a
    # fallback row, which the SWR stale window must not resurrect — a stale fallback re-probes.
    cache: dict = {}
    with patch.object(mod, "_load_provider_models_cache", return_value=cache), \
         patch.object(mod, "_credential_fingerprint", return_value="fp"), \
         patch.object(mod, "_save_provider_models_cache"), \
         patch.object(mod, "provider_model_ids", return_value=mod.CuratedFallbackModels(STATIC)):
        assert mod.cached_provider_model_ids("deepinfra") == STATIC
    assert cache["deepinfra"]["fallback"] is True

    # Same row, now past the fallback TTL: the live fetch runs instead of the stale-serve path,
    # and a real catalog replaces the fallback row.
    cache["deepinfra"]["at"] = time.time() - mod._PROVIDER_MODELS_FALLBACK_TTL - 1
    with patch.object(mod, "_load_provider_models_cache", return_value=cache), \
         patch.object(mod, "_credential_fingerprint", return_value="fp"), \
         patch.object(mod, "_save_provider_models_cache"), \
         patch.object(mod, "_spawn_swr_refresh") as spawn, \
         patch.object(mod, "provider_model_ids", return_value=list(LIVE)) as live:
        assert mod.cached_provider_model_ids("deepinfra") == LIVE
    live.assert_called_once()
    spawn.assert_not_called()
    assert cache["deepinfra"]["models"] == LIVE
    assert "fallback" not in cache["deepinfra"]


def test_codex_cold_fallback_retries_after_short_ttl(monkeypatch, tmp_path):
    from hermes_cli import auth, codex_models

    monkeypatch.setenv("CODEX_HOME", str(tmp_path))
    monkeypatch.setattr(auth, "resolve_codex_runtime_credentials", lambda **_: {"api_key": "test-token"})
    monkeypatch.setattr(auth, "_codex_access_token_is_expiring", lambda *_: False)
    responses = iter([[], ["gpt-6-astra"]])
    monkeypatch.setattr(codex_models, "_fetch_models_from_api", lambda *_args, **_kw: next(responses))
    cache = {}
    with patch.object(mod, "_load_provider_models_cache", return_value=cache), \
         patch.object(mod, "_credential_fingerprint", return_value="fp"), \
         patch.object(mod, "_save_provider_models_cache"), \
         patch.object(mod, "_spawn_swr_refresh") as spawn:
        mod.cached_provider_model_ids("openai-codex")
        assert cache["openai-codex"]["fallback"] is True
        cache["openai-codex"]["at"] = time.time() - mod._PROVIDER_MODELS_FALLBACK_TTL - 1
        assert "gpt-6-astra" in mod.cached_provider_model_ids("openai-codex")
        assert "fallback" not in cache["openai-codex"]
    spawn.assert_not_called()


def test_anthropic_catalog_marks_the_curated_list_as_fallback_on_a_failed_live_fetch():
    # _anthropic_catalog used to `return curated` (a plain list) on a failed live fetch, so a
    # transient outage on a proxy base_url cached the generic curated list over the account's/
    # proxy's real catalog for a full hour (same class of bug as #107391, just missed by 3fb8b3f6).
    with patch.object(mod, "_get_model_config_dict", return_value={}), \
         patch.object(mod, "_fetch_anthropic_models", return_value=None):
        rows = mod.provider_model_ids("anthropic")
    assert isinstance(rows, mod.CuratedFallbackModels)
    assert rows == list(mod._PROVIDER_MODELS["anthropic"])