The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(config-remote): a config backend outage ends one request, fire or turn, never the process
Ben Barclayf4c54cd3ccb9
diff --git a/cron/jobs.py b/cron/jobs.pyindex 3b26b6305a3..1226d06d30d 100644--- a/cron/jobs.py+++ b/cron/jobs.py@@ -33,6 +33,7 @@ from hermes_constants import get_hermes_homefrom cron.constants import CLAIM_TTL_INACTIVITY_HEADROOM, FIRE_CLAIM_SKEW_SECONDS, FIRE_CLAIM_TTL_SECONDSfrom cron.env_settings import cron_env_settingfrom cron.jobs_invariants import _validate_job_mode_invariants+from cron.jobs_model_pin import _main_model_pinfrom cron.scheduler_ownership import _claim_owner_is_deadfrom cron import store_healthfrom typing import Optional, Dict, List, Any, Callable, Set, Tuple, Union, Collection@@ -1652,26 +1653,6 @@ def _normalize_workdir(workdir: Optional[str]) -> Optional[str]:return str(resolved)-def _main_model_pin() -> tuple[Optional[str], Optional[str]]:- """``(provider, model)`` the main agent runs on right now (``model.default`` + the provider it- resolves to), for ``pinned=True`` jobs: the lock is a plain per-job pin, so the scheduler needs- no second precedence axis. ``(None, None)`` when nothing is configured (the job stays unpinned)."""- from hermes_cli.config_backend import config_exists- from hermes_cli.config_effective import load_user_config_effective- cfg_path = get_hermes_home() / "config.yaml"- cfg = load_user_config_effective(cfg_path) if config_exists(cfg_path) else {}- model_cfg = cfg.get("model") or {}- model = model_cfg.get("default") or model_cfg.get("model") if isinstance(model_cfg, dict) else model_cfg- model = _normalize_job_optional_text(model)- if not model:- return None, None- provider = None- with contextlib.suppress(Exception):- from hermes_cli.runtime_provider import resolve_runtime_provider- provider = _normalize_job_optional_text(resolve_runtime_provider(requested=None).get("provider"))- return (provider.lower() if provider else None), model--def _normalize_job_optional_text(value: Any, *, strip_trailing_slash: bool = False) -> Optional[str]:diff --git a/cron/jobs_model_pin.py b/cron/jobs_model_pin.pynew file mode 100644index 00000000000..c721573a798--- /dev/null+++ b/cron/jobs_model_pin.py@@ -0,0 +1,36 @@+"""The main agent's current ``(provider, model)``, which ``pinned=True`` locks onto a cron job."""++from __future__ import annotations++import contextlib+from typing import Optional++from hermes_constants import get_hermes_home+++def _main_model_pin() -> tuple[Optional[str], Optional[str]]:+ """``(provider, model)`` the main agent runs on right now (``model.default`` + the provider it+ resolves to), for ``pinned=True`` jobs: the lock is a plain per-job pin, so the scheduler needs+ no second precedence axis. ``(None, None)`` when nothing is configured (the job stays unpinned).++ ``ValueError`` when the config backend cannot serve the profile (``ConfigBackendUnavailable``):+ answering "nothing configured" would silently drop the lock the caller asked for, so the+ create/update is refused like any other invalid input, and the caller's process lives on."""+ from cron.jobs import _normalize_job_optional_text # late: the facade imports this module+ from hermes_cli.config_backend import ConfigBackendUnavailable, config_exists+ from hermes_cli.config_effective import load_user_config_effective+ cfg_path = get_hermes_home() / "config.yaml"+ try:+ cfg = load_user_config_effective(cfg_path) if config_exists(cfg_path) else {}+ except ConfigBackendUnavailable as exc:+ raise ValueError(f"cannot pin the job to the main model: {exc.code}") from exc+ model_cfg = cfg.get("model") or {}+ model = model_cfg.get("default") or model_cfg.get("model") if isinstance(model_cfg, dict) else model_cfg+ model = _normalize_job_optional_text(model)+ if not model:+ return None, None+ provider = None+ with contextlib.suppress(Exception):+ from hermes_cli.runtime_provider import resolve_runtime_provider+ provider = _normalize_job_optional_text(resolve_runtime_provider(requested=None).get("provider"))+ return (provider.lower() if provider else None), modeldiff --git a/cron/scheduler.py b/cron/scheduler.pyindex 838a4c62e7e..35b8c338215 100644--- a/cron/scheduler.py+++ b/cron/scheduler.py@@ -43,6 +43,7 @@ from cron.env_settings import cron_env_settingfrom hermes_cli._subprocess_compat import windows_hide_flagsfrom hermes_cli.config import (load_config, load_config_readonly)+from hermes_cli.config_backend import ConfigBackendUnavailablefrom hermes_cli.fallback_config import get_fallback_chain, scoped_fallback_chainfrom hermes_time import now as _hermes_now, safe_strftimefrom agent.interrupt_compat import request_hard_interrupt@@ -1651,7 +1652,7 @@ def _load_cron_job_config(job: dict, job_id: str, job_name: str) -> _CronJobConf_model_cfg.get("default") or _model_cfg.get("model") or _model_cfg.get("name")if isinstance(_model_cfg, dict) else "")model = str(_main or "").strip() or model- except Exception as e:+ except Exception as e: # ConfigBackendUnavailable is not one: run_job fails this fire, never defaultslogger.warning("Job '%s': failed to load config.yaml, using defaults: %s", job_id, e)model = (model if isinstance(model, str) else "").strip() or trial_model()# Fail fast: an empty model otherwise reaches the provider as an opaque 400 (#23979).@@ -2648,7 +2649,7 @@ def run_job(_audit.write(dict(result, response_silent=_is_cron_silence_response(final_response or "")), None)return True, output, final_response, None- except Exception as e:+ except (Exception, ConfigBackendUnavailable) as e: # the plane being down fails THIS fire onlyerror_msg = f"{type(e).__name__}: {e!s}"logger.exception("Job '%s' failed: %s", job_name, error_msg)# Cowork-style unreachable-model re-run (cron/unreachable_retry.py): flag failures where@@ -3445,6 +3446,8 @@ def _run_one_job_body(# and no error. Owner fencing still applies: a stale worker must not record over a replacement claim# owner._err_text = str(e) or type(e).__name__+ # A config backend outage (a SystemExit by design) fails this fire, never the ticker.+ contained = isinstance(e, (Exception, ConfigBackendUnavailable))logger.error("Error processing job %s: %s",job["id"],@@ -3454,7 +3457,7 @@ def _run_one_job_body(# Owner fencing: a stale worker whose claim was taken over (or transport-cancelled) must not# send a failure alert on top of the replacement run's; fall through to fenced bookkeeping.if (- isinstance(e, Exception)+ containedand not delivery_attemptedand not isinstance(e, _FireClaimLostDuringSideEffect)and not _fire_claim_ownership_lost()@@ -3469,7 +3472,7 @@ def _run_one_job_body(mark_kwargs = {}if fire_owner is not None:mark_kwargs["expected_fire_owner"] = fire_owner- if isinstance(e, Exception):+ if contained:mark_kwargs["delivery_error"] = delivery_errorfrom cron.unreachable_retry import is_retry_runif is_retry_run(job):@@ -3484,7 +3487,7 @@ def _run_one_job_body(execution_id, success=False, error=_err_text, delivery_outcome=delivery_outcome)except Exception as record_err:logger.error("Failed to finish execution record for job %s: %s", job["id"], record_err)- if not isinstance(e, Exception):+ if not contained:raisereturn Falsefinally:diff --git a/gateway/platforms/api_server.py b/gateway/platforms/api_server.pyindex 88e4fd9e41d..90b020e8c5b 100644--- a/gateway/platforms/api_server.py+++ b/gateway/platforms/api_server.py@@ -236,8 +236,7 @@ def _clean_request_string(value: Any) -> Optional[str]:# model_options decoding lives in the topical sibling (line-cap offset);# re-exported here so importers are unaffected.-from gateway.platforms.api_server_request_options import (- _request_reasoning_config, _request_service_tier)+from gateway.platforms.api_server_request_options import _request_reasoning_config, _request_service_tierdef _apply_runtime_agent_overrides(@@ -3964,9 +3963,10 @@ class APIServerAdapter(OpenAICompatRoutesMixin, BasePlatformAdapter):retryable=False)return Falsetry:- mws = [mw for mw in (- self._make_profile_prefix_middleware(), cors_middleware, body_limit_middleware,- security_headers_middleware) if mw is not None]+ from gateway.platforms.api_server_config_guard import config_unavailable_middleware+ mws = [mw for mw in ( # the config guard outermost: entering a profile scope reads config+ config_unavailable_middleware, self._make_profile_prefix_middleware(), cors_middleware,+ body_limit_middleware, security_headers_middleware) if mw is not None]self._app = web.Application(middlewares=mws, client_max_size=MAX_REQUEST_BYTES)assert self._app is not None# Native routes + multiplex /p/<profile>/ mirrors (the prefix middleware validates anddiff --git a/gateway/platforms/api_server_config_guard.py b/gateway/platforms/api_server_config_guard.pynew file mode 100644index 00000000000..9a526f4344c--- /dev/null+++ b/gateway/platforms/api_server_config_guard.py@@ -0,0 +1,23 @@+"""aiohttp middleware answering a config-backend outage instead of letting it end the gateway.++``ConfigBackendUnavailable`` is a ``SystemExit`` on purpose (config readers must not serve+defaults instead of the selected backend's config). Escaping an aiohttp handler task, a+``SystemExit`` stops the event loop, so a remote-config plane blip during one request took the+whole gateway down. This middleware turns it into that request's 503.+"""++from __future__ import annotations++from aiohttp import web++from hermes_cli.config_backend import ConfigBackendUnavailable+++async def config_unavailable_middleware(request: web.Request, handler):+ try:+ return await handler(request)+ except ConfigBackendUnavailable as exc:+ return web.json_response(+ {"error": {"message": str(exc.code), "type": "server_error", "code": "config_backend_unavailable"}},+ status=503)diff --git a/gateway/platforms/base.py b/gateway/platforms/base.pyindex cb3d51dc76f..016a74b9e02 100644--- a/gateway/platforms/base.py+++ b/gateway/platforms/base.py@@ -4577,8 +4577,8 @@ class BasePlatformAdapter(ABC):await self._run_processing_hook("on_processing_complete", event, ProcessingOutcome.FAILURE)logger.error("[%s] Error handling message: %s", self.name, e, exc_info=True)_thread_metadata = (await self._notify_turn_error(event, e)) or _thread_metadata- # SystemExit/KeyboardInterrupt propagate; other BaseExceptions are contained.- if isinstance(e, (SystemExit, KeyboardInterrupt)):+ from hermes_cli.config_backend import ConfigBackendUnavailable as _outage # this turn only+ if isinstance(e, (SystemExit, KeyboardInterrupt)) and not isinstance(e, _outage): # rest containedraisefinally:await self._release_turn_marker(event)diff --git a/gateway/readiness.py b/gateway/readiness.pyindex 651aab21028..2b31cfa9ead 100644--- a/gateway/readiness.py+++ b/gateway/readiness.py@@ -49,12 +49,15 @@ def _probe_state_db(home: Path) -> dict[str, Any]:def _probe_config(home: Path) -> dict[str, Any]:- from hermes_cli.config_backend import config_exists, read_config_doc_readonly+ from hermes_cli.config_backend import ConfigBackendUnavailable, config_exists, read_config_doc_readonlypath = home / "config.yaml"- if not config_exists(path):- return _check("ok", "using defaults")try:+ if not config_exists(path):+ return _check("ok", "using defaults")raw = read_config_doc_readonly(path)+ except ConfigBackendUnavailable:+ # A SystemExit on purpose (readers must not serve defaults); this probe exists to report it.+ return _check("degraded", "config backend unavailable")except Exception as exc:return _check("degraded", f"invalid config ({type(exc).__name__})")return _check("ok") if raw is None or isinstance(raw, dict) else _check("degraded", "top level is not a mapping")diff --git a/gateway/run_profile_reconcile.py b/gateway/run_profile_reconcile.pyindex 59dd4cf7af5..db79c2a7d0a 100644--- a/gateway/run_profile_reconcile.py+++ b/gateway/run_profile_reconcile.py@@ -27,6 +27,7 @@ from pathlib import Pathfrom typing import Any, Dict, Optionalfrom gateway.run_shutdown import _log_suppressed+from hermes_cli.config_backend import ConfigBackendUnavailablefrom utils import file_cache_versionlogger = logging.getLogger(__name__)@@ -120,7 +121,7 @@ class GatewayProfileReconcileMixin:await self.reconcile_served_profiles(reason="watcher")except asyncio.CancelledError:raise- except Exception:+ except (Exception, ConfigBackendUnavailable): # a config outage is a SystemExit by designlogger.warning("Served-profile reconcile failed; retrying next cycle", exc_info=True)async def _sync_hooks_on_consent_change(self) -> None:@@ -409,7 +410,7 @@ class GatewayProfileReconcileMixin:return Nonetry:await serve_profile_runtime(self, name, home)- except Exception as exc:+ except (Exception, ConfigBackendUnavailable) as exc: # unservable now: parked, retried by rescanif self.session_authorities.for_home(home) is not None:# Retirement of the failed attempt missed its deadline: its writer may still use# this home, so the reservation and authority stay; the profile is parked (nodiff --git a/hermes_cli/config_read_errors.py b/hermes_cli/config_read_errors.pyindex 5bf73ed12ef..b1f31d21fbc 100644--- a/hermes_cli/config_read_errors.py+++ b/hermes_cli/config_read_errors.py@@ -129,7 +129,7 @@ class FailedConfigRead(dict):after one transient EMFILE/EIO replaced the whole file with the fallback plus the caller's edit.A dict subclass so the refusal survives the load→mutate→save round trip at every call site."""- def __init__(self, data: Any = (), *, error: Exception):+ def __init__(self, data: Any = (), *, error: BaseException):super().__init__(data)self.read_error = error@@ -146,6 +146,8 @@ def _refuse_failed_read(config_path: Path, data: Any) -> None:exc = data.read_errorif isinstance(exc, OSError):raise _refuse_overwrite(config_path, "could not be read", exc, _read_error_fix(exc))+ if isinstance(exc, SystemExit): # ConfigBackendUnavailable: the config backend could not serve it+ raise RuntimeError(f"The config could not be read, so this change was not saved: {exc.code}")raise _refuse_overwrite(config_path, "has a formatting error", exc, _FIX_YAML.format(backups=_backups_dir_display()))diff --git a/tests/cron/test_cron_provider_pin.py b/tests/cron/test_cron_provider_pin.pyindex 153f908c686..59090885d40 100644--- a/tests/cron/test_cron_provider_pin.py+++ b/tests/cron/test_cron_provider_pin.py@@ -173,6 +173,8 @@ class TestPinnedLocksTheMainModel:from cron import jobs(tmp_path / "config.yaml").write_text(f"model:\n default: {main_model}\n")monkeypatch.setattr(jobs, "get_hermes_home", lambda: tmp_path, raising=True)+ from cron import jobs_model_pin # the pin reads the main model's config where it lives+ monkeypatch.setattr(jobs_model_pin, "get_hermes_home", lambda: tmp_path, raising=True)state = {"jobs": []}monkeypatch.setattr(jobs, "load_jobs", lambda: list(state["jobs"]), raising=True)monkeypatch.setattr(jobs, "save_jobs", lambda j: state.__setitem__("jobs", list(j)), raising=True)diff --git a/tests/plugins/config_backends/test_remote_unavailable_consumers.py b/tests/plugins/config_backends/test_remote_unavailable_consumers.pynew file mode 100644index 00000000000..cef3ba69949--- /dev/null+++ b/tests/plugins/config_backends/test_remote_unavailable_consumers.py@@ -0,0 +1,240 @@+"""andrexibiza P1-B: ``ConfigBackendUnavailable`` is a ``SystemExit`` on purpose (a fail-open reader+must never serve defaults instead of the selected backend's config; boot fails closed), so an+``except Exception`` / ``suppress(Exception)`` around a config read does not catch it. Each+long-lived consumer below handles it explicitly with ITS outcome, and its process keeps running.++Every test raises the real ``ConfigBackendUnavailable`` from the config-read seam the consumer+actually calls and asserts the contained outcome. Each one fails (the exit propagates) with its+fix reverted.+"""+from __future__ import annotations++import asyncio+import threading+from pathlib import Path++import pytest++from hermes_cli.config_backend import ConfigBackendUnavailable++OUTAGE = "Remote Config: cannot load the config for profile 'default' from https://plane: HTTP 503"+++def _outage(*_a, **_kw):+ raise ConfigBackendUnavailable(OUTAGE)+++# --- the six named sites ------------------------------------------------------------------------++def test_readiness_probe_reports_config_backend_unavailable(tmp_path, monkeypatch):+ """The probe whose job is to report config health reports the outage, never crashes."""+ from gateway import readiness+ monkeypatch.setattr("hermes_cli.config_backend.config_exists", _outage)+ assert readiness._probe_config(tmp_path) == {"status": "degraded", "detail": "config backend unavailable"}++ monkeypatch.setattr("hermes_cli.config_backend.config_exists", lambda _p: True)+ monkeypatch.setattr("hermes_cli.config_backend.read_config_doc_readonly", _outage)+ assert readiness._probe_config(tmp_path)["status"] == "degraded"++ monkeypatch.setenv("HERMES_HOME", str(tmp_path))+ result = readiness.collect_runtime_readiness(configured_model="m", runtime_status={})+ assert result["status"] == "degraded" and result["checks"]["config"]["status"] == "degraded"+++def test_cron_tick_fails_only_that_fire_and_never_runs_on_defaults(tmp_path, monkeypatch):+ """``_load_cron_job_config`` keeps refusing defaults (the SystemExit passes its ``except+ Exception``); ``run_job`` — the fire's owner — turns it into this fire's failure result."""+ import run_agent # run_job imports it; its import-time dotenv load reads config+ from cron import scheduler+ monkeypatch.setattr("hermes_cli.config_backend.config_exists", _outage)+ with pytest.raises(ConfigBackendUnavailable):+ scheduler._load_cron_job_config({"id": "j", "model": "m"}, "j", "job")++ # Through the real run_job as the fire's owner: the pre-agent gates pass, then the read refuses.+ monkeypatch.setattr("gateway.session_cron.current_execution", lambda: (None, "s", "j", "fire"))+ monkeypatch.setattr(scheduler, "_prepare_job_prompt", lambda *a, **k: (None, "do the thing"))+ monkeypatch.setattr(scheduler, "_reload_dotenv_and_publish_delivery_target", lambda job: None)+ monkeypatch.setattr(scheduler, "_get_hermes_home", lambda: tmp_path)+ success, doc, final, error = scheduler.run_job({"id": "j", "name": "job", "prompt": "p"},+ execution_id="fire")+ assert success is False and final == ""+ assert error and "ConfigBackendUnavailable" in error and "HTTP 503" in error+ assert "FAILED" in doc+++def test_cron_job_body_records_the_outage_and_keeps_the_ticker(monkeypatch, tmp_path):+ """The body owner (``_run_one_job_body``) records a fire that escaped with the outage as failed+ and returns, instead of re-raising it into the ticker thread like a real SystemExit."""+ from cron import executions, jobs, scheduler+ monkeypatch.setattr(scheduler, "run_job", lambda job, **kw: _outage())+ marked = []+ real_mark = scheduler.mark_job_run+ monkeypatch.setattr(scheduler, "mark_job_run", lambda *a, **kw: (marked.append((a, kw)), real_mark(*a, **kw)))+ with jobs.use_cron_store(tmp_path / "cron"):+ job = jobs.create_job(prompt="report", schedule="every 1h", deliver="local")+ fire = executions.create_execution(job["id"], source="test")["id"]+ assert scheduler._run_one_job_body(dict(jobs.get_job(job["id"]), execution_id=fire)) is False+ assert executions.get_execution(fire)["status"] == "failed"+ assert marked and marked[0][0][1] is False and "HTTP 503" in marked[0][0][2]+ assert jobs.get_job(job["id"])["last_status"] == "error"+++def test_cron_pin_refuses_instead_of_dropping_the_lock(monkeypatch, tmp_path):+ """``pinned=True`` reads the main model; with the plane down that is an invalid request+ (ValueError, the create/update path's refusal), never ``(None, None)`` = silently unpinned+ and never a SystemExit out of the cronjob tool or RPC."""+ from cron import jobs+ from cron.jobs_model_pin import _main_model_pin+ monkeypatch.setattr("hermes_cli.config_backend.config_exists", _outage)+ with pytest.raises(ValueError, match="cannot pin the job to the main model.*HTTP 503"):+ _main_model_pin()+ with jobs.use_cron_store(tmp_path / "cron"):+ with pytest.raises(ValueError, match="cannot pin"):+ jobs.create_job(prompt="report", schedule="every 1h", deliver="local", pinned=True)+ assert jobs.load_jobs() == []+++def test_tui_profile_cwd_returns_none(monkeypatch, tmp_path):+ from tui_gateway import server+ monkeypatch.setattr(server, "config_exists", _outage)+ assert server._profile_configured_cwd(tmp_path) is None+ assert server._default_session_cwd() # the session-create tail still answers a cwd+++def test_tui_raw_config_read_is_a_failed_read_that_refuses_to_save(monkeypatch, tmp_path):+ """``_load_cfg_raw`` answers its designed degraded value (``FailedConfigRead``) and the+ write-back primitive refuses to persist it, so the RPC answers an error instead of dying+ or saving a ``{}`` over the plane's document."""+ from hermes_cli.config_read_errors import FailedConfigRead, _refuse_failed_read+ from tui_gateway import server+ server._drop_cfg_cache()+ monkeypatch.setattr(server, "config_version", _outage)+ monkeypatch.setattr(server, "config_exists", lambda _p: True)+ data = server._load_cfg_raw()+ assert isinstance(data, FailedConfigRead) and data == {}+ assert isinstance(data.read_error, ConfigBackendUnavailable)+ with pytest.raises(RuntimeError) as refused:+ _refuse_failed_read(tmp_path / "config.yaml", data)+ assert "could not be read" in str(refused.value) and "HTTP 503" in str(refused.value)+ assert "formatting error" not in str(refused.value) # not a YAML problem; nothing to edit+++def test_prompt_attachments_degrade_to_the_profile_home_store(monkeypatch, tmp_path):+ """``file.attach`` keeps working: storage falls back to ``""`` (the profile-home store the+ sandbox mounts and the vision allowlist always cover)."""+ from tui_gateway import server+ monkeypatch.setattr("hermes_cli.config_backend.config_exists", _outage)+ assert server._profile_attachments_storage(tmp_path) == ""+ session = {"profile_home": str(tmp_path), "cwd": str(tmp_path)}+ assert server._session_home_dir(session, "attachments") == tmp_path / "attachments"+++# --- the owners above those sites (the rest of the class) --------------------------------------++def test_tui_pooled_rpc_answers_once_with_the_outage(monkeypatch):+ """The request owner: a long handler that hits the outage still writes one error frame."""+ from tui_gateway import server+ monkeypatch.setitem(server._methods, "cc.refused", lambda rid, params: _outage())+ monkeypatch.setattr(server, "_LONG_HANDLERS", server._LONG_HANDLERS | {"cc.refused"})+ written = threading.Event()+ frames = []++ class Recorder:+ def write(self, obj):+ frames.append(obj)+ written.set()+ return True++ def close(self):+ pass++ assert server.dispatch({"jsonrpc": "2.0", "id": "r1", "method": "cc.refused", "params": {}}, Recorder()) is None+ assert written.wait(10)+ assert frames[0]["id"] == "r1" and "HTTP 503" in frames[0]["error"]["message"]+++def test_tui_agent_build_thread_reports_the_outage(monkeypatch):+ """The deferred agent-build thread records ``agent_error`` and releases its waiters (a+ SystemExit would have ended the thread with ``agent_ready`` never set)."""+ from tui_gateway import server+ sid = "build-outage"+ session = {"session_key": "k", "agent": None, "agent_ready": threading.Event(),+ "history_lock": threading.Lock(), "profile_home": None}+ monkeypatch.setitem(server._sessions, sid, session)+ monkeypatch.setattr(server, "_await_resume_history", lambda sid, current: True)+ monkeypatch.setattr(server, "_bind_build_profile_scopes", lambda home: None)+ monkeypatch.setattr(server, "_make_agent", lambda *a, **k: _outage())+ monkeypatch.setattr(server, "_deferred_build_agent_kwargs", lambda current, db: {})+ monkeypatch.setattr(server, "_emit", lambda *a, **k: None)+ monkeypatch.setattr("tui_gateway.entry.ensure_mcp_discovery_started", lambda: None, raising=False)+ server._start_agent_build(sid, session)+ session["_agent_build_thread"].join(10)+ assert session["agent_ready"].is_set()+ assert "HTTP 503" in str(session.get("agent_error"))+++def test_gateway_turn_owner_contains_the_outage_and_notifies():+ """A platform turn that hits the outage notifies the user and ends that turn only: re-raised,+ a SystemExit out of the background task stops the gateway's event loop."""+ from tests.gateway.test_baseexception_turn_notify import _event, _make_adapter, _raising_handler+ from gateway.session import build_session_key+ adapter = _make_adapter(_raising_handler(ConfigBackendUnavailable(OUTAGE)))+ event = _event()+ asyncio.run(adapter._process_message_background(event, build_session_key(event.source)))+ assert adapter.sent and "error" in adapter.sent[0].lower()+++def test_gateway_api_server_answers_503():+ pytest.importorskip("aiohttp")+ from aiohttp import web+ from aiohttp.test_utils import TestClient, TestServer+ from gateway.platforms.api_server_config_guard import config_unavailable_middleware++ async def handler(_request):+ _outage()++ async def main():+ app = web.Application(middlewares=[config_unavailable_middleware])+ app.router.add_get("/v1/x", handler)+ async with TestClient(TestServer(app)) as client:+ resp = await client.get("/v1/x")+ return resp.status, await resp.json()++ status, body = asyncio.run(main())+ assert status == 503 and body["error"]["code"] == "config_backend_unavailable"+++def test_gateway_profile_reconcile_parks_the_profile(monkeypatch, tmp_path):+ """Serving a new multiplexed profile whose first config fetch fails parks that profile (the+ rescan retries it) instead of ending the reconcile watcher and the loop."""+ from gateway import run_profile_reconcile as rpr+ released = []++ class Runner(rpr.GatewayProfileReconcileMixin):+ session_authorities = type("A", (), {"for_home": staticmethod(lambda home: None)})()++ async def serve(runner, name, home):+ _outage()++ monkeypatch.setattr("gateway.run_runtime.reserve_profile_home", lambda *a: None)+ monkeypatch.setattr("gateway.run_runtime.release_profile_home", lambda runner, home: released.append(home))+ monkeypatch.setattr("gateway.run_runtime.serve_profile_runtime", serve)+ verdict = asyncio.run(Runner()._serve_profile_runtime("work", Path(tmp_path)))+ assert verdict and "HTTP 503" in verdict and released == [Path(tmp_path)]+++def test_compute_host_turn_answers_turn_error(monkeypatch):+ """The dashboard compute host's turn worker answers ``turn.error`` and keeps serving."""+ import io+ import json as _json+ from tui_gateway.compute_host import ComputeHost+ out = io.StringIO()+ host = ComputeHost(stdout=out, heartbeat_secs=0)+ monkeypatch.setattr(host, "_ensure_server_session", lambda server, frame: _outage())+ try:+ host._run_real_turn({"sid": "s1", "request_id": "q1", "text": "hi"})+ finally:+ host.close()+ frames = [_json.loads(line) for line in out.getvalue().splitlines()]+ assert [f["type"] for f in frames] == ["turn.error"]+ assert frames[0]["request_id"] == "q1" and "HTTP 503" in frames[0]["message"]diff --git a/tui_gateway/compute_host.py b/tui_gateway/compute_host.pyindex 78a609a3be7..9c8b398a025 100644--- a/tui_gateway/compute_host.py+++ b/tui_gateway/compute_host.py@@ -24,6 +24,7 @@ import uuidfrom pathlib import Pathfrom typing import Any, Callable, Collection+from hermes_cli.config_backend import ConfigBackendUnavailablefrom tui_gateway.host_supervisor import MUTATOR_ROUTE_TABLE, _build_sha@@ -186,7 +187,7 @@ class ComputeHost:try:from tui_gateway import serverbody(server, sid, request_id)- except Exception as exc:+ except (Exception, ConfigBackendUnavailable) as exc: # health: allow BLE001 -- the frame's one reply (the outage is a SystemExit)if on_error is not None:on_error(sid)self._reply(error_kind, sid, request_id, **error_extra, message=str(exc))@@ -291,7 +292,7 @@ class ComputeHost:self._reply("turn.end", sid, request_id, **meta, interrupted=interrupted, ended_ns=now_ns(),session_info=session_info, session_info_emitted=True)- except Exception as exc:+ except (Exception, ConfigBackendUnavailable) as exc: # health: allow BLE001 -- the turn's one reply (the outage is a SystemExit)with contextlib.suppress(Exception):from tui_gateway import serversession = server._sessions.get(sid)diff --git a/tui_gateway/prompt_attachments.py b/tui_gateway/prompt_attachments.pyindex d8b8e4e7b0e..b43006ae5e8 100644--- a/tui_gateway/prompt_attachments.py+++ b/tui_gateway/prompt_attachments.py@@ -128,17 +128,25 @@ def _profile_attachments_storage(profile_home) -> str:Read from THAT profile's config.yaml — ``file.attach`` runs before ``prompt.submit``installs the profile scope, so the process config still belongs to the launch profile- (same reason as ``_profile_configured_cwd``)."""+ (same reason as ``_profile_configured_cwd``).++ Best-effort, including when the config backend cannot serve the profile: ``""`` keeps the+ profile-home store, which the sandbox mounts and the vision allowlist always cover, so the+ attachment still reaches the agent (by an absolute instead of a workspace-relative ref)."""import contextlib as _contextlib+ from hermes_cli.config_backend import ConfigBackendUnavailablehome = Path(profile_home) if profile_home else _hermes_home- with _contextlib.suppress(Exception):- from hermes_cli.config_backend import config_exists- from hermes_cli.config_effective import load_user_config_effective- cfg_path = home / "config.yaml"- if config_exists(cfg_path):- attachments_cfg = load_user_config_effective(cfg_path).get("attachments")- if isinstance(attachments_cfg, dict):- return str(attachments_cfg.get("storage") or "").strip().lower()+ try:+ with _contextlib.suppress(Exception):+ from hermes_cli.config_backend import config_exists+ from hermes_cli.config_effective import load_user_config_effective+ cfg_path = home / "config.yaml"+ if config_exists(cfg_path):+ attachments_cfg = load_user_config_effective(cfg_path).get("attachments")+ if isinstance(attachments_cfg, dict):+ return str(attachments_cfg.get("storage") or "").strip().lower()+ except ConfigBackendUnavailable as exc:+ logger.warning("attachments.storage for %s unavailable (%s); staging in the profile home", home, exc.code)return ""diff --git a/tui_gateway/server.py b/tui_gateway/server.pyindex 9f45a54d44f..01f364ebbc7 100644--- a/tui_gateway/server.py+++ b/tui_gateway/server.py@@ -28,7 +28,7 @@ from hermes_constants import (get_hermes_home, get_hermes_home_override, get_process_hermes_home, profile_name_for_home,reset_hermes_home_override, set_hermes_home_override)from hermes_cli import config_backend as _config_backend-from hermes_cli.config_backend import Changes, config_exists, supports_file_tooling+from hermes_cli.config_backend import Changes, ConfigBackendUnavailable, config_exists, supports_file_toolingfrom hermes_cli.env_loader import load_hermes_dotenvfrom tui_gateway.pet_sheet import _clone_pet_payload, _pet_row_frame_counts, _pet_sheet_revision, _pet_state_rowsfrom utils import is_truthy_value@@ -676,10 +676,13 @@ def _profile_configured_cwd(profile_home: Path | None) -> str | None:"""if profile_home is None:return None- with contextlib.suppress(Exception):- from hermes_cli.config_effective import load_user_config_effective- p = Path(profile_home) / "config.yaml"- return _configured_cwd_from_cfg(load_user_config_effective(p)) if config_exists(p) else None+ try:+ with contextlib.suppress(Exception):+ from hermes_cli.config_effective import load_user_config_effective+ p = Path(profile_home) / "config.yaml"+ return _configured_cwd_from_cfg(load_user_config_effective(p)) if config_exists(p) else None+ except ConfigBackendUnavailable as exc: # a SystemExit: the agent build reports it; the cwd is a hint+ logger.warning("terminal.cwd of %s unavailable: %s", profile_home, exc.code)return None@@ -1013,7 +1016,7 @@ def _load_cfg_raw() -> dict:if _cfg_cache is not None and _cfg_sig == sig and _cfg_path == p:return copy.deepcopy(_cfg_cache)data = read_user_config_raw(p) if config_exists(p) else {}- except Exception as exc:+ except (Exception, ConfigBackendUnavailable) as exc: # health: allow BLE001 -- served as a FailedConfigRead, refused by _save_cfgreturn FailedConfigRead(error=exc) # readable as {}, refused by _save_cfgwith _cfg_lock: # cache the RAW config: _save_cfg writes _cfg_cache back to disk_cfg_cache, _cfg_sig, _cfg_path = copy.deepcopy(data), sig, pdiff --git a/tui_gateway/session_registry.py b/tui_gateway/session_registry.pyindex 4d96df74c31..c85cd7831ae 100644--- a/tui_gateway/session_registry.py+++ b/tui_gateway/session_registry.py@@ -281,7 +281,7 @@ def _start_agent_build(sid: str, session: dict) -> None:# MCP fleet, and live-transport sessions are never reaped, so fleets would accumulate.notify_registered = _wire_session_agent(sid, key, agent)_announce_built_agent(sid, key, current, agent)- except Exception as e:+ except (Exception, ConfigBackendUnavailable) as e: # health: allow BLE001 -- reported as agent_error + error eventfrom agent.auxiliary_unavailable import ProviderNotConfiguredErrorcurrent["agent_error"] = str(e)# A client can route "no provider is set up" to its setup flow instead of a dead-end@@ -617,7 +617,7 @@ def _schedule_resume_hydration(sid: str, stored_id: str, db, *, close_db: bool ={"message_count": session["resume_message_count"], "phase": "history", "status": "complete"})_maybe_schedule_auto_continue(sid, session, stored_id)_start_agent_build(sid, session)- except Exception as exc:+ except (Exception, ConfigBackendUnavailable) as exc: # health: allow BLE001 -- reported as the resume failureif _sessions.get(sid) is not session:returnmessage = resume_failed_message(exc)