The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(config): the provider env catalog reads no config
Ben Barclayb4d8e4c3322e
diff --git a/hermes_cli/config_provider_env.py b/hermes_cli/config_provider_env.pyindex bb7d1f1cc13..9fa841714fe 100644--- a/hermes_cli/config_provider_env.py+++ b/hermes_cli/config_provider_env.py@@ -1,12 +1,11 @@-"""The provider-profile entries of ``OPTIONAL_ENV_VARS``, added on first read instead of at import.--Listing provider profiles runs provider discovery, and discovery reads config: ``plugins.enabled``-gates pip-installed provider plugins, ``plugins.isolation`` decides how ``$HERMES_HOME`` ones load.-Done while ``hermes_cli.config`` imports, that read is the launch profile's config in a process that-serves several profiles, and a cycle with a config backend that imports this code while it loads the-config (the remote backend's Nous credential imports ``hermes_cli.auth``, which imports-``hermes_cli.config``). So importing ``hermes_cli.config`` reads no config; the first read of the-table (or of ``CORE_DECLARED_ENV_NAMES``) adds the entries.+"""The provider-profile entries of ``OPTIONAL_ENV_VARS``: descriptive metadata that reads no config.++They used to come from ``list_providers()`` while ``hermes_cli.config`` imported, and provider+discovery reads config (``plugins.enabled``, ``plugins.isolation``): the launch profile's config in+a process serving several profiles, and a deadlock with a config backend whose own load imports+``hermes_cli.auth``. Now the bundled providers' entries are built from :func:`providers.bundled_profiles`+(no config, no discovery) on the table's first read, and discovery's completion hook adds the+process-wide plugin providers it registered."""from __future__ import annotations@@ -21,22 +20,18 @@ def arm(table, static_names) -> None:table.arm(_provider_entries, _PROVIDER_NAMES.update)-def _provider_entries() -> dict:- """The env vars of every ``auth_type="api_key"`` provider profile. A core entry wins; a- provider entry replaces one a platform plugin manifest added (it was added first only because- this runs late); the first profile to declare a name wins."""- try:- from providers import list_providers- profiles = list_providers()- except Exception: # health: allow BLE001 -- as before: a broken provider layer must not break config readers- # (ConfigBackendUnavailable is a SystemExit: it propagates, and the table stays armed)- return {}+_CATALOG: dict | None = None # the bundled providers' entries: config-free, computed once+++def _entries_for(profiles, skip) -> dict:+ """Entries for the ``auth_type="api_key"`` profiles' env vars, minus ``skip``; the first profile+ to declare a name wins."""entries: dict = {}for pp in profiles:if pp.auth_type != "api_key":continuefor var in pp.env_vars:- if var in _STATIC_NAMES or var in entries:+ if var in skip or var in entries:continueis_key = not var.endswith(("_BASE_URL", "_URL"))label = pp.display_name or pp.name@@ -50,6 +45,35 @@ def _provider_entries() -> dict:return entries+def _provider_entries() -> dict:+ """The bundled provider profiles' entries. Reads no config and runs no discovery (so it has no+ partial state and nothing to wait for): a plain idempotent cache, and concurrent first callers+ just compute the same result. A core entry wins; a provider entry replaces one a platform plugin+ manifest added (it was added first only because this runs late)."""+ global _CATALOG+ if _CATALOG is None:+ try:+ from providers import bundled_profiles+ profiles = bundled_profiles()+ except Exception: # health: allow BLE001 -- as before: a broken provider layer must not break config readers+ profiles = []+ _CATALOG = _entries_for(profiles, _STATIC_NAMES)+ return dict(_CATALOG)+++def sync_registered_providers() -> None:+ """Provider discovery's completion hook: add the env vars of process-wide plugin providers it+ registered (entry points enabled in config). Reads the registry as it stands, never config;+ ``$HERMES_HOME`` plugins (per profile) are not process-wide and are left out."""+ import providers+ from hermes_cli.config_defaults import OPTIONAL_ENV_VARS+ profiles = [p for name, p in list(providers._REGISTRY.items()) if providers.provider_source(name) != "bundled"]+ entries = _entries_for(profiles, _STATIC_NAMES | _PROVIDER_NAMES | set(_CATALOG or ()))+ for var, entry in entries.items():+ dict.__setitem__(OPTIONAL_ENV_VARS, var, entry)+ _PROVIDER_NAMES.update(entries)++def core_declared_env_names() -> frozenset[str]:"""Names declared in core: config_defaults' plus the provider profiles'. A platform manifestnever reclassifies one."""diff --git a/providers/__init__.py b/providers/__init__.pyindex f24c98fed28..f824ed0d64e 100644--- a/providers/__init__.py+++ b/providers/__init__.py@@ -100,6 +100,8 @@ _HOME_LAYERS_LOCK = threading.Lock()# across the import itself — a thread mid-``import hermes_cli.auth`` (whose import calls# ``list_providers()``) would block on it while the scanning thread waits on that module's import lock._REGISTRATION_TARGET: ContextVar[_HomeLayer | None] = ContextVar("_provider_registration_target", default=None)+# Set by :func:`bundled_profiles`: registrations made in this context are collected, nothing else.+_COLLECT: ContextVar[list | None] = ContextVar("_provider_collect", default=None)# Profile names an in-process import may register (None = all). Set while a hosted plugin's# profiles that cannot cross the host boundary are imported here: its hosted siblings stay hosted._REGISTRATION_ONLY: ContextVar[frozenset[str] | None] = ContextVar("_provider_registration_only", default=None)@@ -124,6 +126,7 @@ def _sync_auth_registry() -> None:for module, attr in (("hermes_cli.auth", "sync_plugin_provider_registry"),("hermes_cli.models_catalog_static", "sync_plugin_provider_catalog"),+ ("hermes_cli.config_provider_env", "sync_registered_providers"),):sync = getattr(sys.modules.get(module), attr, None)if sync is None:@@ -143,6 +146,10 @@ def register_provider(profile: ProviderProfile) -> None:``$HERMES_HOME`` plugin is being imported lands in that home's layer."""global _PROVIDER_LIST_CACHE, _REGISTRY_GENERATION+ collect = _COLLECT.get()+ if collect is not None:+ collect.append(profile)+ returnonly = _REGISTRATION_ONLY.get()if only is not None and profile.name not in only:return@@ -819,6 +826,33 @@ def _requires_arguments(fn) -> bool:return False+def bundled_profiles() -> list[ProviderProfile]:+ """Every bundled provider profile, one per name (the last registration wins, in discovery+ order), WITHOUT config and without touching the registry: each bundled plugin is executed into+ a private module whose registrations are collected. For descriptive catalogs, which must not+ depend on config or on discovery having run."""+ collected: list[ProviderProfile] = []+ token = _COLLECT.set(collected)+ try:+ for child in sorted(_BUNDLED_PLUGINS_DIR.iterdir()) if _BUNDLED_PLUGINS_DIR.is_dir() else ():+ init = child / "__init__.py"+ if not child.is_dir() or child.name.startswith(("_", ".")) or not init.exists():+ continue+ name = f"_hermes_provider_catalog_{child.name.replace('-', '_')}"+ spec = importlib.util.spec_from_file_location(name, init, submodule_search_locations=[str(child)])+ if spec is None or spec.loader is None:+ continue+ module = importlib.util.module_from_spec(spec)+ sys.modules[name] = module # relative imports inside the plugin+ try:+ spec.loader.exec_module(module)+ except Exception as exc: # health: allow BLE001 -- one broken plugin never breaks the catalog+ logger.debug("provider catalog skipped %s: %s", child.name, exc)+ finally:+ _COLLECT.reset(token)+ return list({p.name: p for p in collected}.values())++def _discover_providers() -> None:"""Populate the process-wide registry by importing every provider plugin.diff --git a/tests/hermes_cli/test_config_provider_env.py b/tests/hermes_cli/test_config_provider_env.pyindex 5a1bf792f20..49c0a29bdb2 100644--- a/tests/hermes_cli/test_config_provider_env.py+++ b/tests/hermes_cli/test_config_provider_env.py@@ -1,4 +1,4 @@-"""Provider-profile entries of OPTIONAL_ENV_VARS: added on first read, same contents as before."""+"""Provider-profile entries of OPTIONAL_ENV_VARS: config-free, same contents as before."""from __future__ import annotationsfrom types import SimpleNamespace@@ -25,10 +25,39 @@ def test_every_api_key_provider_var_is_declared_in_core():assert expected <= cfg.CORE_DECLARED_ENV_NAMES+def test_bundled_profiles_match_discovery_and_leave_the_registry_alone():+ import providers++ before = dict(providers._REGISTRY)+ bundled = {p.name: p for p in providers.bundled_profiles()}+ assert dict(providers._REGISTRY) == before+ discovered = {n for n in providers._REGISTRY if providers.provider_source(n) == "bundled"}+ assert discovered and set(bundled) >= discovered+ for name in discovered:+ assert bundled[name].env_vars == providers._REGISTRY[name].env_vars+++def test_the_catalog_builds_under_a_backend_that_refuses_every_read(monkeypatch):+ import hermes_cli.config_backend as cb++ reads = []++ def refuse(*a, **k):+ reads.append(a)+ raise ConfigBackendUnavailable("this test's backend refuses every read")++ monkeypatch.setattr(cb, "_route", refuse)+ monkeypatch.setattr(cpe, "_CATALOG", None)+ entries = cpe._provider_entries()+ assert "ANTHROPIC_API_KEY" in entries or len(entries) > 10+ assert reads == []++@pytest.fixturedef table(monkeypatch):monkeypatch.setattr(cpe, "_STATIC_NAMES", frozenset({"CORE_KEY"}))monkeypatch.setattr(cpe, "_PROVIDER_NAMES", set())+ monkeypatch.setattr(cpe, "_CATALOG", None)t = OptionalEnvVars({"CORE_KEY": {"description": "core"}, "PLUG_KEY": {"description": "platform"}})t.arm(cpe._provider_entries, cpe._PROVIDER_NAMES.update)return t@@ -40,7 +69,7 @@ def test_fill_runs_on_first_read_core_wins_provider_replaces_platform_first_provcalls = []profiles = [_profile("alpha", "CORE_KEY", "PLUG_KEY", "ALPHA_API_KEY", "ALPHA_BASE_URL"),_profile("beta", "ALPHA_API_KEY"), _profile("oauth", "OAUTH_TOKEN", auth_type="oauth")]- monkeypatch.setattr(providers, "list_providers", lambda: calls.append(1) or profiles)+ monkeypatch.setattr(providers, "bundled_profiles", lambda: calls.append(1) or profiles)dict.__setitem__(table, "LATE", {}) # writes and raw dict access never fillassert calls == []assert table["CORE_KEY"] == {"description": "core"} and calls == [1]@@ -51,32 +80,18 @@ def test_fill_runs_on_first_read_core_wins_provider_replaces_platform_first_provassert calls == [1] # once-def test_unavailable_config_fails_the_read_loudly_and_the_next_read_retries(table, monkeypatch):+def test_discovery_completion_adds_registered_plugin_providers(table, monkeypatch):import providers- answers = [ConfigBackendUnavailable("boot in flight"), [_profile("alpha", "ALPHA_API_KEY")]]-- def list_providers():- answer = answers.pop(0)- if isinstance(answer, BaseException):- raise answer- return answer-- monkeypatch.setattr(providers, "list_providers", list_providers)- with pytest.raises(ConfigBackendUnavailable): # fail closed, never a silently partial table- table.get("ALPHA_API_KEY")- assert "ALPHA_API_KEY" in table and answers == []---def test_a_broken_provider_layer_is_not_retried(table, monkeypatch):- import providers-- calls = []-- def boom():- calls.append(1)- raise RuntimeError("broken")-- monkeypatch.setattr(providers, "list_providers", boom)- assert "CORE_KEY" in table and "CORE_KEY" in table- assert calls == [1]+ from hermes_cli import config_defaults++ monkeypatch.setattr(providers, "bundled_profiles", lambda: [_profile("alpha", "ALPHA_API_KEY")])+ monkeypatch.setattr(config_defaults, "OPTIONAL_ENV_VARS", table)+ monkeypatch.setattr(providers, "_REGISTRY", {"alpha": _profile("alpha", "ALPHA_API_KEY"),+ "plug": _profile("plug", "PLUG_API_KEY", "ALPHA_API_KEY")})+ monkeypatch.setattr(providers, "provider_source", lambda n: "bundled" if n == "alpha" else "entry_point")+ assert "ALPHA_API_KEY" in table+ cpe.sync_registered_providers()+ assert table["PLUG_API_KEY"]["description"] == "Plug API key"+ assert table["ALPHA_API_KEY"]["description"] == "Alpha API key" # an earlier entry is kept+ assert "PLUG_API_KEY" in cpe._PROVIDER_NAMESdiff --git a/tests/test_state_config_contention.py b/tests/test_state_config_contention.pyindex b2650e25130..fb8cecde9ba 100644--- a/tests/test_state_config_contention.py+++ b/tests/test_state_config_contention.py@@ -18,38 +18,21 @@ from utils import atomic_yaml_writeimport hermes_state_wal as wal-def test_first_provider_env_fill_cannot_invert_config_and_wal_locks(tmp_path):- """Importing hermes_cli.config lists no providers (that reads config); the first read of- OPTIONAL_ENV_VARS does, holding no lock, while another thread opens a database."""+def test_config_import_and_provider_env_catalog_never_read_config(tmp_path):+ """Importing hermes_cli.config and building the provider part of OPTIONAL_ENV_VARS list no+ providers through discovery (which reads config), so neither can take part in a config/WAL+ lock-order cycle with a thread opening a database."""script = textwrap.dedent("""from pathlib import Path- from threading import Event, Thread- import os, sqlite3, sys+ from threading import Thread+ import os, sqlite3import providersimport hermes_state_wal as wal- partial_ready = Event()- opener_inside_resolve = Event()- importer_load_returned = Event()- original_resolve = wal.resolve_journal_mode-- def controlled_list_providers():- module = sys.modules["hermes_cli.config"]- assert not module.__spec__._initializing, "hermes_cli.config listed providers at import"- partial_ready.set()- assert opener_inside_resolve.wait(5)- module.load_config()- importer_load_returned.set()- return []-- def controlled_resolve():- opener_inside_resolve.set()- return original_resolve()-- providers.list_providers = controlled_list_providers- wal.resolve_journal_mode = controlled_resolve+ listed = []+ providers.list_providers = lambda: listed.append(1) or []root = Path(os.environ["STATE_CONFIG_TEST_ROOT"]) / "cold-import"root.mkdir()os.environ["HOME"] = str(root)@@ -61,26 +44,23 @@ def test_first_provider_env_fill_cannot_invert_config_and_wal_locks(tmp_path):def importer():import hermes_cli.config- assert not partial_ready.is_set()outcomes.append("import-returned")- hermes_cli.config.OPTIONAL_ENV_VARS.get("OPENAI_API_KEY") # the first read fills+ assert "ANTHROPIC_API_KEY" in hermes_cli.config.OPTIONAL_ENV_VARSoutcomes.append("fill-returned")def opener():with sqlite3.connect(database) as connection:outcomes.append(f"open:{wal.apply_wal_with_fallback(connection)}")- importing = Thread(target=importer)- opening = Thread(target=opener)- importing.start()- assert partial_ready.wait(5)- opening.start()- importing.join(5)- opening.join(5)- assert importer_load_returned.is_set()- assert not importing.is_alive() and not opening.is_alive(), outcomes- assert "import-returned" in outcomes and "fill-returned" in outcomes+ threads = [Thread(target=importer), Thread(target=opener)]+ for t in threads:+ t.start()+ for t in threads:+ t.join(10)+ assert not any(t.is_alive() for t in threads), outcomes+ assert "import-returned" in outcomes and "fill-returned" in outcomes, outcomesassert any(value.startswith("open:") for value in outcomes)+ assert listed == [], "the config import or the env catalog ran provider discovery"""")completed = subprocess.run(