Skip to content

The agent that grows with you

fix(config): the provider env catalog reads no config

Ben Barclayb4d8e4c3322e
diff --git a/hermes_cli/config_provider_env.py b/hermes_cli/config_provider_env.py
index bb7d1f1cc13..9fa841714fe 100644
--- a/hermes_cli/config_provider_env.py
+++ b/hermes_cli/config_provider_env.py
@@ -1,12 +1,11 @@
-"""The provider-profile entries of ``OPTIONAL_ENV_VARS``, added on first read instead of at import.
-
-Listing provider profiles runs provider discovery, and discovery reads config: ``plugins.enabled``
-gates pip-installed provider plugins, ``plugins.isolation`` decides how ``$HERMES_HOME`` ones load.
-Done while ``hermes_cli.config`` imports, that read is the launch profile's config in a process that
-serves several profiles, and a cycle with a config backend that imports this code while it loads the
-config (the remote backend's Nous credential imports ``hermes_cli.auth``, which imports
-``hermes_cli.config``). So importing ``hermes_cli.config`` reads no config; the first read of the
-table (or of ``CORE_DECLARED_ENV_NAMES``) adds the entries.
+"""The provider-profile entries of ``OPTIONAL_ENV_VARS``: descriptive metadata that reads no config.
+
+They used to come from ``list_providers()`` while ``hermes_cli.config`` imported, and provider
+discovery reads config (``plugins.enabled``, ``plugins.isolation``): the launch profile's config in
+a process serving several profiles, and a deadlock with a config backend whose own load imports
+``hermes_cli.auth``. Now the bundled providers' entries are built from :func:`providers.bundled_profiles`
+(no config, no discovery) on the table's first read, and discovery's completion hook adds the
+process-wide plugin providers it registered.
"""
from __future__ import annotations
@@ -21,22 +20,18 @@ def arm(table, static_names) -> None:
table.arm(_provider_entries, _PROVIDER_NAMES.update)
-def _provider_entries() -> dict:
- """The env vars of every ``auth_type="api_key"`` provider profile. A core entry wins; a
- provider entry replaces one a platform plugin manifest added (it was added first only because
- this runs late); the first profile to declare a name wins."""
- try:
- from providers import list_providers
- profiles = list_providers()
- except Exception: # health: allow BLE001 -- as before: a broken provider layer must not break config readers
- # (ConfigBackendUnavailable is a SystemExit: it propagates, and the table stays armed)
- return {}
+_CATALOG: dict | None = None # the bundled providers' entries: config-free, computed once
+
+
+def _entries_for(profiles, skip) -> dict:
+ """Entries for the ``auth_type="api_key"`` profiles' env vars, minus ``skip``; the first profile
+ to declare a name wins."""
entries: dict = {}
for pp in profiles:
if pp.auth_type != "api_key":
continue
for var in pp.env_vars:
- if var in _STATIC_NAMES or var in entries:
+ if var in skip or var in entries:
continue
is_key = not var.endswith(("_BASE_URL", "_URL"))
label = pp.display_name or pp.name
@@ -50,6 +45,35 @@ def _provider_entries() -> dict:
return entries
+def _provider_entries() -> dict:
+ """The bundled provider profiles' entries. Reads no config and runs no discovery (so it has no
+ partial state and nothing to wait for): a plain idempotent cache, and concurrent first callers
+ just compute the same result. A core entry wins; a provider entry replaces one a platform plugin
+ manifest added (it was added first only because this runs late)."""
+ global _CATALOG
+ if _CATALOG is None:
+ try:
+ from providers import bundled_profiles
+ profiles = bundled_profiles()
+ except Exception: # health: allow BLE001 -- as before: a broken provider layer must not break config readers
+ profiles = []
+ _CATALOG = _entries_for(profiles, _STATIC_NAMES)
+ return dict(_CATALOG)
+
+
+def sync_registered_providers() -> None:
+ """Provider discovery's completion hook: add the env vars of process-wide plugin providers it
+ registered (entry points enabled in config). Reads the registry as it stands, never config;
+ ``$HERMES_HOME`` plugins (per profile) are not process-wide and are left out."""
+ import providers
+ from hermes_cli.config_defaults import OPTIONAL_ENV_VARS
+ profiles = [p for name, p in list(providers._REGISTRY.items()) if providers.provider_source(name) != "bundled"]
+ entries = _entries_for(profiles, _STATIC_NAMES | _PROVIDER_NAMES | set(_CATALOG or ()))
+ for var, entry in entries.items():
+ dict.__setitem__(OPTIONAL_ENV_VARS, var, entry)
+ _PROVIDER_NAMES.update(entries)
+
+
def core_declared_env_names() -> frozenset[str]:
"""Names declared in core: config_defaults' plus the provider profiles'. A platform manifest
never reclassifies one."""
diff --git a/providers/__init__.py b/providers/__init__.py
index f24c98fed28..f824ed0d64e 100644
--- a/providers/__init__.py
+++ b/providers/__init__.py
@@ -100,6 +100,8 @@ _HOME_LAYERS_LOCK = threading.Lock()
# across the import itself — a thread mid-``import hermes_cli.auth`` (whose import calls
# ``list_providers()``) would block on it while the scanning thread waits on that module's import lock.
_REGISTRATION_TARGET: ContextVar[_HomeLayer | None] = ContextVar("_provider_registration_target", default=None)
+# Set by :func:`bundled_profiles`: registrations made in this context are collected, nothing else.
+_COLLECT: ContextVar[list | None] = ContextVar("_provider_collect", default=None)
# Profile names an in-process import may register (None = all). Set while a hosted plugin's
# profiles that cannot cross the host boundary are imported here: its hosted siblings stay hosted.
_REGISTRATION_ONLY: ContextVar[frozenset[str] | None] = ContextVar("_provider_registration_only", default=None)
@@ -124,6 +126,7 @@ def _sync_auth_registry() -> None:
for module, attr in (
("hermes_cli.auth", "sync_plugin_provider_registry"),
("hermes_cli.models_catalog_static", "sync_plugin_provider_catalog"),
+ ("hermes_cli.config_provider_env", "sync_registered_providers"),
):
sync = getattr(sys.modules.get(module), attr, None)
if sync is None:
@@ -143,6 +146,10 @@ def register_provider(profile: ProviderProfile) -> None:
``$HERMES_HOME`` plugin is being imported lands in that home's layer.
"""
global _PROVIDER_LIST_CACHE, _REGISTRY_GENERATION
+ collect = _COLLECT.get()
+ if collect is not None:
+ collect.append(profile)
+ return
only = _REGISTRATION_ONLY.get()
if only is not None and profile.name not in only:
return
@@ -819,6 +826,33 @@ def _requires_arguments(fn) -> bool:
return False
+def bundled_profiles() -> list[ProviderProfile]:
+ """Every bundled provider profile, one per name (the last registration wins, in discovery
+ order), WITHOUT config and without touching the registry: each bundled plugin is executed into
+ a private module whose registrations are collected. For descriptive catalogs, which must not
+ depend on config or on discovery having run."""
+ collected: list[ProviderProfile] = []
+ token = _COLLECT.set(collected)
+ try:
+ for child in sorted(_BUNDLED_PLUGINS_DIR.iterdir()) if _BUNDLED_PLUGINS_DIR.is_dir() else ():
+ init = child / "__init__.py"
+ if not child.is_dir() or child.name.startswith(("_", ".")) or not init.exists():
+ continue
+ name = f"_hermes_provider_catalog_{child.name.replace('-', '_')}"
+ spec = importlib.util.spec_from_file_location(name, init, submodule_search_locations=[str(child)])
+ if spec is None or spec.loader is None:
+ continue
+ module = importlib.util.module_from_spec(spec)
+ sys.modules[name] = module # relative imports inside the plugin
+ try:
+ spec.loader.exec_module(module)
+ except Exception as exc: # health: allow BLE001 -- one broken plugin never breaks the catalog
+ logger.debug("provider catalog skipped %s: %s", child.name, exc)
+ finally:
+ _COLLECT.reset(token)
+ return list({p.name: p for p in collected}.values())
+
+
def _discover_providers() -> None:
"""Populate the process-wide registry by importing every provider plugin.
diff --git a/tests/hermes_cli/test_config_provider_env.py b/tests/hermes_cli/test_config_provider_env.py
index 5a1bf792f20..49c0a29bdb2 100644
--- a/tests/hermes_cli/test_config_provider_env.py
+++ b/tests/hermes_cli/test_config_provider_env.py
@@ -1,4 +1,4 @@
-"""Provider-profile entries of OPTIONAL_ENV_VARS: added on first read, same contents as before."""
+"""Provider-profile entries of OPTIONAL_ENV_VARS: config-free, same contents as before."""
from __future__ import annotations
from types import SimpleNamespace
@@ -25,10 +25,39 @@ def test_every_api_key_provider_var_is_declared_in_core():
assert expected <= cfg.CORE_DECLARED_ENV_NAMES
+def test_bundled_profiles_match_discovery_and_leave_the_registry_alone():
+ import providers
+
+ before = dict(providers._REGISTRY)
+ bundled = {p.name: p for p in providers.bundled_profiles()}
+ assert dict(providers._REGISTRY) == before
+ discovered = {n for n in providers._REGISTRY if providers.provider_source(n) == "bundled"}
+ assert discovered and set(bundled) >= discovered
+ for name in discovered:
+ assert bundled[name].env_vars == providers._REGISTRY[name].env_vars
+
+
+def test_the_catalog_builds_under_a_backend_that_refuses_every_read(monkeypatch):
+ import hermes_cli.config_backend as cb
+
+ reads = []
+
+ def refuse(*a, **k):
+ reads.append(a)
+ raise ConfigBackendUnavailable("this test's backend refuses every read")
+
+ monkeypatch.setattr(cb, "_route", refuse)
+ monkeypatch.setattr(cpe, "_CATALOG", None)
+ entries = cpe._provider_entries()
+ assert "ANTHROPIC_API_KEY" in entries or len(entries) > 10
+ assert reads == []
+
+
@pytest.fixture
def table(monkeypatch):
monkeypatch.setattr(cpe, "_STATIC_NAMES", frozenset({"CORE_KEY"}))
monkeypatch.setattr(cpe, "_PROVIDER_NAMES", set())
+ monkeypatch.setattr(cpe, "_CATALOG", None)
t = OptionalEnvVars({"CORE_KEY": {"description": "core"}, "PLUG_KEY": {"description": "platform"}})
t.arm(cpe._provider_entries, cpe._PROVIDER_NAMES.update)
return t
@@ -40,7 +69,7 @@ def test_fill_runs_on_first_read_core_wins_provider_replaces_platform_first_prov
calls = []
profiles = [_profile("alpha", "CORE_KEY", "PLUG_KEY", "ALPHA_API_KEY", "ALPHA_BASE_URL"),
_profile("beta", "ALPHA_API_KEY"), _profile("oauth", "OAUTH_TOKEN", auth_type="oauth")]
- monkeypatch.setattr(providers, "list_providers", lambda: calls.append(1) or profiles)
+ monkeypatch.setattr(providers, "bundled_profiles", lambda: calls.append(1) or profiles)
dict.__setitem__(table, "LATE", {}) # writes and raw dict access never fill
assert calls == []
assert table["CORE_KEY"] == {"description": "core"} and calls == [1]
@@ -51,32 +80,18 @@ def test_fill_runs_on_first_read_core_wins_provider_replaces_platform_first_prov
assert calls == [1] # once
-def test_unavailable_config_fails_the_read_loudly_and_the_next_read_retries(table, monkeypatch):
+def test_discovery_completion_adds_registered_plugin_providers(table, monkeypatch):
import providers
- answers = [ConfigBackendUnavailable("boot in flight"), [_profile("alpha", "ALPHA_API_KEY")]]
-
- def list_providers():
- answer = answers.pop(0)
- if isinstance(answer, BaseException):
- raise answer
- return answer
-
- monkeypatch.setattr(providers, "list_providers", list_providers)
- with pytest.raises(ConfigBackendUnavailable): # fail closed, never a silently partial table
- table.get("ALPHA_API_KEY")
- assert "ALPHA_API_KEY" in table and answers == []
-
-
-def test_a_broken_provider_layer_is_not_retried(table, monkeypatch):
- import providers
-
- calls = []
-
- def boom():
- calls.append(1)
- raise RuntimeError("broken")
-
- monkeypatch.setattr(providers, "list_providers", boom)
- assert "CORE_KEY" in table and "CORE_KEY" in table
- assert calls == [1]
+ from hermes_cli import config_defaults
+
+ monkeypatch.setattr(providers, "bundled_profiles", lambda: [_profile("alpha", "ALPHA_API_KEY")])
+ monkeypatch.setattr(config_defaults, "OPTIONAL_ENV_VARS", table)
+ monkeypatch.setattr(providers, "_REGISTRY", {"alpha": _profile("alpha", "ALPHA_API_KEY"),
+ "plug": _profile("plug", "PLUG_API_KEY", "ALPHA_API_KEY")})
+ monkeypatch.setattr(providers, "provider_source", lambda n: "bundled" if n == "alpha" else "entry_point")
+ assert "ALPHA_API_KEY" in table
+ cpe.sync_registered_providers()
+ assert table["PLUG_API_KEY"]["description"] == "Plug API key"
+ assert table["ALPHA_API_KEY"]["description"] == "Alpha API key" # an earlier entry is kept
+ assert "PLUG_API_KEY" in cpe._PROVIDER_NAMES
diff --git a/tests/test_state_config_contention.py b/tests/test_state_config_contention.py
index b2650e25130..fb8cecde9ba 100644
--- a/tests/test_state_config_contention.py
+++ b/tests/test_state_config_contention.py
@@ -18,38 +18,21 @@ from utils import atomic_yaml_write
import hermes_state_wal as wal
-def test_first_provider_env_fill_cannot_invert_config_and_wal_locks(tmp_path):
- """Importing hermes_cli.config lists no providers (that reads config); the first read of
- OPTIONAL_ENV_VARS does, holding no lock, while another thread opens a database."""
+def test_config_import_and_provider_env_catalog_never_read_config(tmp_path):
+ """Importing hermes_cli.config and building the provider part of OPTIONAL_ENV_VARS list no
+ providers through discovery (which reads config), so neither can take part in a config/WAL
+ lock-order cycle with a thread opening a database."""
script = textwrap.dedent(
"""
from pathlib import Path
- from threading import Event, Thread
- import os, sqlite3, sys
+ from threading import Thread
+ import os, sqlite3
import providers
import hermes_state_wal as wal
- partial_ready = Event()
- opener_inside_resolve = Event()
- importer_load_returned = Event()
- original_resolve = wal.resolve_journal_mode
-
- def controlled_list_providers():
- module = sys.modules["hermes_cli.config"]
- assert not module.__spec__._initializing, "hermes_cli.config listed providers at import"
- partial_ready.set()
- assert opener_inside_resolve.wait(5)
- module.load_config()
- importer_load_returned.set()
- return []
-
- def controlled_resolve():
- opener_inside_resolve.set()
- return original_resolve()
-
- providers.list_providers = controlled_list_providers
- wal.resolve_journal_mode = controlled_resolve
+ listed = []
+ providers.list_providers = lambda: listed.append(1) or []
root = Path(os.environ["STATE_CONFIG_TEST_ROOT"]) / "cold-import"
root.mkdir()
os.environ["HOME"] = str(root)
@@ -61,26 +44,23 @@ def test_first_provider_env_fill_cannot_invert_config_and_wal_locks(tmp_path):
def importer():
import hermes_cli.config
- assert not partial_ready.is_set()
outcomes.append("import-returned")
- hermes_cli.config.OPTIONAL_ENV_VARS.get("OPENAI_API_KEY") # the first read fills
+ assert "ANTHROPIC_API_KEY" in hermes_cli.config.OPTIONAL_ENV_VARS
outcomes.append("fill-returned")
def opener():
with sqlite3.connect(database) as connection:
outcomes.append(f"open:{wal.apply_wal_with_fallback(connection)}")
- importing = Thread(target=importer)
- opening = Thread(target=opener)
- importing.start()
- assert partial_ready.wait(5)
- opening.start()
- importing.join(5)
- opening.join(5)
- assert importer_load_returned.is_set()
- assert not importing.is_alive() and not opening.is_alive(), outcomes
- assert "import-returned" in outcomes and "fill-returned" in outcomes
+ threads = [Thread(target=importer), Thread(target=opener)]
+ for t in threads:
+ t.start()
+ for t in threads:
+ t.join(10)
+ assert not any(t.is_alive() for t in threads), outcomes
+ assert "import-returned" in outcomes and "fill-returned" in outcomes, outcomes
assert any(value.startswith("open:") for value in outcomes)
+ assert listed == [], "the config import or the env catalog ran provider discovery"
"""
)
completed = subprocess.run(