The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
"""First-run onboarding routing for a completely unconfigured install.
Regression tests for the "keyless first run boots into a broken chat" bug:
a fresh install with zero providers accepted a message, spun for ~30s, then
failed with a provider-specific error ("Set OPENROUTER_API_KEY") the user
never chose, and never offered setup.
Covers:
- ``_runtime_credentials_ready()`` silent probe semantics
- ``_offer_first_run_setup()`` routing into the shared provider picker
- the provider-aware (non-OpenRouter-specific) empty-key error message
"""
import importlib
import os
import sys
import types
import pytest
from hermes_cli.auth import AuthError
from agent.i18n import t
def _reset_modules(prefixes: tuple[str, ...]):
for name in list(sys.modules):
if any(name == p or name.startswith(p + ".") for p in prefixes):
sys.modules.pop(name, None)
@pytest.fixture(autouse=True)
def _restore_cli_and_tool_modules():
prefixes = ("tools", "cli", "run_agent")
original_modules = {
name: module
for name, module in sys.modules.items()
if any(name == p or name.startswith(p + ".") for p in prefixes)
}
try:
yield
finally:
_reset_modules(prefixes)
sys.modules.update(original_modules)
def _import_cli():
for name in list(sys.modules):
if name == "cli" or name == "run_agent" or name == "tools" or name.startswith("tools."):
sys.modules.pop(name, None)
if "firecrawl" not in sys.modules:
sys.modules["firecrawl"] = types.SimpleNamespace(Firecrawl=object)
return importlib.import_module("cli")
def _make_shell(cli, monkeypatch):
shell = cli.HermesCLI(compact=True, max_turns=1)
return shell
# ---------------------------------------------------------------------------
# _runtime_credentials_ready
# ---------------------------------------------------------------------------
def test_credentials_ready_false_when_no_provider(monkeypatch):
cli = _import_cli()
def _raise(**kwargs):
raise AuthError("No inference provider configured.", code="no_provider_configured")
monkeypatch.setattr("hermes_cli.runtime_provider.resolve_runtime_provider", _raise)
shell = _make_shell(cli, monkeypatch)
assert shell._runtime_credentials_ready() is False
def test_credentials_ready_false_on_empty_openrouter_key(monkeypatch):
"""The exact broken-chat state: provider resolves but api_key is empty."""
cli = _import_cli()
monkeypatch.setattr(
"hermes_cli.runtime_provider.resolve_runtime_provider",
lambda **kw: {
"provider": "openrouter",
"api_key": "",
"base_url": "https://openrouter.ai/api/v1",
"source": "env/config",
},
)
shell = _make_shell(cli, monkeypatch)
assert shell._runtime_credentials_ready() is False
def test_credentials_ready_true_with_key(monkeypatch):
cli = _import_cli()
monkeypatch.setattr(
"hermes_cli.runtime_provider.resolve_runtime_provider",
lambda **kw: {
"provider": "openrouter",
"api_key": "sk-test",
"base_url": "https://openrouter.ai/api/v1",
"source": "env/config",
},
)
shell = _make_shell(cli, monkeypatch)
assert shell._runtime_credentials_ready() is True
def test_credentials_ready_true_for_keyless_local_endpoint(monkeypatch):
"""ollama/llama.cpp-style custom endpoints need no key."""
cli = _import_cli()
monkeypatch.setattr(
"hermes_cli.runtime_provider.resolve_runtime_provider",
lambda **kw: {
"provider": "custom",
"api_key": "",
"base_url": "http://localhost:11434/v1",
"source": "custom_provider",
},
)
shell = _make_shell(cli, monkeypatch)
assert shell._runtime_credentials_ready() is True
def test_credentials_ready_true_for_callable_bearer_provider(monkeypatch):
cli = _import_cli()
monkeypatch.setattr(
"hermes_cli.runtime_provider.resolve_runtime_provider",
lambda **kw: {
"provider": "azure-foundry",
"api_key": lambda: "tok",
"base_url": "https://foundry.example/v1",
"source": "entra",
},
)
shell = _make_shell(cli, monkeypatch)
assert shell._runtime_credentials_ready() is True
def test_credentials_ready_never_prints(monkeypatch, capsys):
cli = _import_cli()
def _raise(**kwargs):
raise AuthError("No inference provider configured.", code="no_provider_configured")
monkeypatch.setattr("hermes_cli.runtime_provider.resolve_runtime_provider", _raise)
shell = _make_shell(cli, monkeypatch)
capsys.readouterr() # drain construction output
shell._runtime_credentials_ready()
out = capsys.readouterr()
assert out.out == ""
# ---------------------------------------------------------------------------
# _offer_first_run_setup
# ---------------------------------------------------------------------------
def test_offer_first_run_setup_routes_into_shared_picker(monkeypatch):
cli = _import_cli()
shell = _make_shell(cli, monkeypatch)
picker_calls = {"count": 0}
def _fake_picker():
picker_calls["count"] += 1
monkeypatch.setattr("hermes_cli.main.select_provider_and_model", _fake_picker)
monkeypatch.setattr("builtins.input", lambda *a, **k: "y")
# After the picker "runs", config has a provider and creds resolve.
monkeypatch.setattr(
"hermes_cli.config.load_config",
lambda: {"model": {"provider": "nous", "default": "hermes-4-405b"}},
)
monkeypatch.setattr(
"hermes_cli.runtime_provider.resolve_runtime_provider",
lambda **kw: {
"provider": "nous",
"api_key": "portal-token",
"base_url": "https://inference-api.nousresearch.com/v1",
"source": "oauth",
},
)
assert shell._offer_first_run_setup() is True
assert picker_calls["count"] == 1
assert shell.requested_provider == "nous"
assert shell.model == "hermes-4-405b"
# Agent must be rebuilt with the new credentials on next use.
assert shell.agent is None
def test_offer_first_run_setup_re_resolves_reasoning_for_picked_model(monkeypatch):
"""The picker moves self.model; the CLI-level reasoning_config must follow it before the
lazily built agent inherits the launch model's effort."""
cli = _import_cli()
monkeypatch.setitem(cli.CLI_CONFIG, "agent", {
**cli.CLI_CONFIG.get("agent", {}), "reasoning_effort": "medium",
"reasoning_overrides": {"hermes-4-405b": "high"}})
shell = _make_shell(cli, monkeypatch)
assert shell.reasoning_config["effort"] == "medium"
monkeypatch.setattr("hermes_cli.main.select_provider_and_model", lambda: None)
monkeypatch.setattr("builtins.input", lambda *a, **k: "y")
monkeypatch.setattr("hermes_cli.config.load_config",
lambda: {"model": {"provider": "nous", "default": "hermes-4-405b"}})
monkeypatch.setattr(shell, "_runtime_credentials_ready", lambda: True)
assert shell._offer_first_run_setup() is True
assert shell.model == "hermes-4-405b"
assert shell.reasoning_config["effort"] == "high"
def test_offer_first_run_setup_declined(monkeypatch):
cli = _import_cli()
shell = _make_shell(cli, monkeypatch)
def _fail_picker():
raise AssertionError("picker must not run when declined")
monkeypatch.setattr("hermes_cli.main.select_provider_and_model", _fail_picker)
monkeypatch.setattr("builtins.input", lambda *a, **k: "n")
assert shell._offer_first_run_setup() is False
def test_offer_first_run_setup_picker_cancel_is_graceful(monkeypatch):
cli = _import_cli()
shell = _make_shell(cli, monkeypatch)
def _cancel_picker():
raise KeyboardInterrupt()
monkeypatch.setattr("hermes_cli.main.select_provider_and_model", _cancel_picker)
monkeypatch.setattr("builtins.input", lambda *a, **k: "")
# Empty answer defaults to yes -> picker runs -> cancels -> False, no raise.
assert shell._offer_first_run_setup() is False
# ---------------------------------------------------------------------------
# Provider-aware empty-key error (replaces the OpenRouter-specific one)
# ---------------------------------------------------------------------------
def test_empty_key_error_names_actual_provider(monkeypatch, capsys):
cli = _import_cli()
monkeypatch.setattr(
"hermes_cli.runtime_provider.resolve_runtime_provider",
lambda **kw: {
"provider": "fireworks",
"api_key": "",
"base_url": "https://api.fireworks.ai/inference/v1/extra",
"source": "env/config",
},
)
shell = _make_shell(cli, monkeypatch)
# A custom base_url would get the no-key placeholder; force the
# openrouter-shaped branch by pointing base_url at openrouter.
monkeypatch.setattr(
"hermes_cli.runtime_provider.resolve_runtime_provider",
lambda **kw: {
"provider": "fireworks",
"api_key": "",
"base_url": "https://openrouter.ai/api/v1",
"source": "env/config",
},
)
capsys.readouterr()
assert shell._ensure_runtime_credentials() is False
out = capsys.readouterr().out
assert "fireworks" in out
assert "OPENROUTER_API_KEY" not in out
assert "hermes model" in out or "hermes setup" in out
# ---------------------------------------------------------------------------
# Configured-but-unusable credential: reason + cooldown, never the wizard (#113720)
# ---------------------------------------------------------------------------
def _bench_nous_pool(monkeypatch, **entry_fields):
import time
from agent.credential_pool import STATUS_EXHAUSTED, CredentialPool, PooledCredential
benched = PooledCredential(id="e1", provider="nous", auth_type="oauth", access_token="x",
refresh_token="r", label="portal", source="manual:device_code",
priority=0, last_status=STATUS_EXHAUSTED, last_status_at=time.time() - 5,
**entry_fields)
pool = CredentialPool.__new__(CredentialPool)
monkeypatch.setattr(pool, "has_credentials", lambda: True, raising=False)
monkeypatch.setattr(pool, "has_available", lambda **kw: False, raising=False)
monkeypatch.setattr(pool, "next_available_at", lambda **kw: time.time() + 55, raising=False)
monkeypatch.setattr(pool, "entries", lambda: [benched], raising=False)
monkeypatch.setattr("agent.credential_pool.load_pool", lambda provider: pool)
def _forbid_wizard(monkeypatch, shell):
monkeypatch.setattr("hermes_cli.main.select_provider_and_model",
lambda: (_ for _ in ()).throw(AssertionError("wizard must not run")))
monkeypatch.setattr(shell, "_offer_first_run_setup",
lambda: (_ for _ in ()).throw(AssertionError("wizard must not be offered")))
def test_benched_credential_prints_cooldown_instead_of_wizard(monkeypatch, capsys):
"""A profile whose only credential is cooling down is not a blank install: the interactive
startup gate prints the cooldown (with why and how long) as the headline, without telling the
user to re-authenticate, and never offers the first-run wizard."""
cli = _import_cli()
shell = _make_shell(cli, monkeypatch)
shell.requested_provider = "nous"
def _raise(**kwargs):
raise AuthError("Hermes is not logged into Nous Portal.", provider="nous",
code="nous_auth_missing", relogin_required=True)
monkeypatch.setattr("hermes_cli.runtime_provider.resolve_runtime_provider", _raise)
_bench_nous_pool(monkeypatch, last_error_code=429, last_error_reason="rate_limited")
_forbid_wizard(monkeypatch, shell)
monkeypatch.setattr(sys.stdin, "isatty", lambda: True)
shell._maybe_offer_first_run_setup()
out = capsys.readouterr().out
assert t("cli.startup.first_run_no_provider") not in out
headline = next(line for line in out.splitlines() if line.strip())
assert headline.endswith(t("cli.startup.credential_cooling_down", provider="nous",
cause=t("cli.startup.cooldown_cause_rate_limit"), minutes=1))
assert t("cli.startup.cooldown_cause_token_refresh") not in out
assert "not logged into Nous Portal" in out
assert "re-authenticate" not in out and "hermes model" not in out
def test_auth_json_only_login_explains_instead_of_wizard(monkeypatch, capsys, tmp_path):
"""auth.json-only shape: logged into Nous but no ``model.provider`` (requested "auto"). The
ladder swallows the AuthError and falls through to a keyless OpenRouter fallback; the gate
must still explain the real failure rather than treat the profile as a blank install.
Control: the resolver's ``no_provider_configured`` still reaches the wizard."""
import dataclasses
import hermes_cli.runtime_provider as rp
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
(tmp_path / "config.yaml").write_text("model:\n default: some-model\n", encoding="utf-8")
for key in [k for k in os.environ if k.endswith("_API_KEY")]:
monkeypatch.delenv(key, raising=False)
def _nous_fail():
raise AuthError("Hermes is not logged into Nous Portal.", provider="nous",
code="nous_auth_missing", relogin_required=True)
monkeypatch.setattr(rp, "resolve_provider", lambda *a, **kw: "nous")
monkeypatch.setitem(rp._OAUTH_RUNTIME_PROVIDERS, "nous",
dataclasses.replace(rp._OAUTH_RUNTIME_PROVIDERS["nous"], resolve=_nous_fail))
cli = _import_cli()
shell = _make_shell(cli, monkeypatch)
shell.requested_provider = "auto"
shell._explicit_api_key = None
shell._explicit_base_url = None
_forbid_wizard(monkeypatch, shell)
monkeypatch.setattr(sys.stdin, "isatty", lambda: True)
shell._maybe_offer_first_run_setup()
out = capsys.readouterr().out
assert "not logged into Nous Portal" in out
assert t("cli.startup.first_run_no_provider") not in out
offered = []
monkeypatch.setattr(shell, "_offer_first_run_setup", lambda: offered.append(True) or True)
monkeypatch.setattr("hermes_cli.runtime_provider.resolve_runtime_provider", lambda **kw: (_ for _ in ()).throw(
AuthError("Hermes is not connected to any AI provider yet.", code="no_provider_configured")))
shell._maybe_offer_first_run_setup()
assert offered == [True]
assert "not logged into" not in capsys.readouterr().out