Skip to content

The agent that grows with you

fix(config-remote): a config backend outage ends one request, fire or turn, never the process

Ben Barclayf4c54cd3ccb9
diff --git a/cron/jobs.py b/cron/jobs.py
index 3b26b6305a3..1226d06d30d 100644
--- a/cron/jobs.py
+++ b/cron/jobs.py
@@ -33,6 +33,7 @@ from hermes_constants import get_hermes_home
from cron.constants import CLAIM_TTL_INACTIVITY_HEADROOM, FIRE_CLAIM_SKEW_SECONDS, FIRE_CLAIM_TTL_SECONDS
from cron.env_settings import cron_env_setting
from cron.jobs_invariants import _validate_job_mode_invariants
+from cron.jobs_model_pin import _main_model_pin
from cron.scheduler_ownership import _claim_owner_is_dead
from cron import store_health
from typing import Optional, Dict, List, Any, Callable, Set, Tuple, Union, Collection
@@ -1652,26 +1653,6 @@ def _normalize_workdir(workdir: Optional[str]) -> Optional[str]:
return str(resolved)
-def _main_model_pin() -> tuple[Optional[str], Optional[str]]:
- """``(provider, model)`` the main agent runs on right now (``model.default`` + the provider it
- resolves to), for ``pinned=True`` jobs: the lock is a plain per-job pin, so the scheduler needs
- no second precedence axis. ``(None, None)`` when nothing is configured (the job stays unpinned)."""
- from hermes_cli.config_backend import config_exists
- from hermes_cli.config_effective import load_user_config_effective
- cfg_path = get_hermes_home() / "config.yaml"
- cfg = load_user_config_effective(cfg_path) if config_exists(cfg_path) else {}
- model_cfg = cfg.get("model") or {}
- model = model_cfg.get("default") or model_cfg.get("model") if isinstance(model_cfg, dict) else model_cfg
- model = _normalize_job_optional_text(model)
- if not model:
- return None, None
- provider = None
- with contextlib.suppress(Exception):
- from hermes_cli.runtime_provider import resolve_runtime_provider
- provider = _normalize_job_optional_text(resolve_runtime_provider(requested=None).get("provider"))
- return (provider.lower() if provider else None), model
-
-
def _normalize_job_optional_text(
value: Any, *, strip_trailing_slash: bool = False
) -> Optional[str]:
diff --git a/cron/jobs_model_pin.py b/cron/jobs_model_pin.py
new file mode 100644
index 00000000000..c721573a798
--- /dev/null
+++ b/cron/jobs_model_pin.py
@@ -0,0 +1,36 @@
+"""The main agent's current ``(provider, model)``, which ``pinned=True`` locks onto a cron job."""
+
+from __future__ import annotations
+
+import contextlib
+from typing import Optional
+
+from hermes_constants import get_hermes_home
+
+
+def _main_model_pin() -> tuple[Optional[str], Optional[str]]:
+ """``(provider, model)`` the main agent runs on right now (``model.default`` + the provider it
+ resolves to), for ``pinned=True`` jobs: the lock is a plain per-job pin, so the scheduler needs
+ no second precedence axis. ``(None, None)`` when nothing is configured (the job stays unpinned).
+
+ ``ValueError`` when the config backend cannot serve the profile (``ConfigBackendUnavailable``):
+ answering "nothing configured" would silently drop the lock the caller asked for, so the
+ create/update is refused like any other invalid input, and the caller's process lives on."""
+ from cron.jobs import _normalize_job_optional_text # late: the facade imports this module
+ from hermes_cli.config_backend import ConfigBackendUnavailable, config_exists
+ from hermes_cli.config_effective import load_user_config_effective
+ cfg_path = get_hermes_home() / "config.yaml"
+ try:
+ cfg = load_user_config_effective(cfg_path) if config_exists(cfg_path) else {}
+ except ConfigBackendUnavailable as exc:
+ raise ValueError(f"cannot pin the job to the main model: {exc.code}") from exc
+ model_cfg = cfg.get("model") or {}
+ model = model_cfg.get("default") or model_cfg.get("model") if isinstance(model_cfg, dict) else model_cfg
+ model = _normalize_job_optional_text(model)
+ if not model:
+ return None, None
+ provider = None
+ with contextlib.suppress(Exception):
+ from hermes_cli.runtime_provider import resolve_runtime_provider
+ provider = _normalize_job_optional_text(resolve_runtime_provider(requested=None).get("provider"))
+ return (provider.lower() if provider else None), model
diff --git a/cron/scheduler.py b/cron/scheduler.py
index 838a4c62e7e..35b8c338215 100644
--- a/cron/scheduler.py
+++ b/cron/scheduler.py
@@ -43,6 +43,7 @@ from cron.env_settings import cron_env_setting
from hermes_cli._subprocess_compat import windows_hide_flags
from hermes_cli.config import (
load_config, load_config_readonly)
+from hermes_cli.config_backend import ConfigBackendUnavailable
from hermes_cli.fallback_config import get_fallback_chain, scoped_fallback_chain
from hermes_time import now as _hermes_now, safe_strftime
from agent.interrupt_compat import request_hard_interrupt
@@ -1651,7 +1652,7 @@ def _load_cron_job_config(job: dict, job_id: str, job_name: str) -> _CronJobConf
_model_cfg.get("default") or _model_cfg.get("model") or _model_cfg.get("name")
if isinstance(_model_cfg, dict) else "")
model = str(_main or "").strip() or model
- except Exception as e:
+ except Exception as e: # ConfigBackendUnavailable is not one: run_job fails this fire, never defaults
logger.warning("Job '%s': failed to load config.yaml, using defaults: %s", job_id, e)
model = (model if isinstance(model, str) else "").strip() or trial_model()
# Fail fast: an empty model otherwise reaches the provider as an opaque 400 (#23979).
@@ -2648,7 +2649,7 @@ def run_job(
_audit.write(dict(result, response_silent=_is_cron_silence_response(final_response or "")), None)
return True, output, final_response, None
- except Exception as e:
+ except (Exception, ConfigBackendUnavailable) as e: # the plane being down fails THIS fire only
error_msg = f"{type(e).__name__}: {e!s}"
logger.exception("Job '%s' failed: %s", job_name, error_msg)
# Cowork-style unreachable-model re-run (cron/unreachable_retry.py): flag failures where
@@ -3445,6 +3446,8 @@ def _run_one_job_body(
# and no error. Owner fencing still applies: a stale worker must not record over a replacement claim
# owner.
_err_text = str(e) or type(e).__name__
+ # A config backend outage (a SystemExit by design) fails this fire, never the ticker.
+ contained = isinstance(e, (Exception, ConfigBackendUnavailable))
logger.error(
"Error processing job %s: %s",
job["id"],
@@ -3454,7 +3457,7 @@ def _run_one_job_body(
# Owner fencing: a stale worker whose claim was taken over (or transport-cancelled) must not
# send a failure alert on top of the replacement run's; fall through to fenced bookkeeping.
if (
- isinstance(e, Exception)
+ contained
and not delivery_attempted
and not isinstance(e, _FireClaimLostDuringSideEffect)
and not _fire_claim_ownership_lost()
@@ -3469,7 +3472,7 @@ def _run_one_job_body(
mark_kwargs = {}
if fire_owner is not None:
mark_kwargs["expected_fire_owner"] = fire_owner
- if isinstance(e, Exception):
+ if contained:
mark_kwargs["delivery_error"] = delivery_error
from cron.unreachable_retry import is_retry_run
if is_retry_run(job):
@@ -3484,7 +3487,7 @@ def _run_one_job_body(
execution_id, success=False, error=_err_text, delivery_outcome=delivery_outcome)
except Exception as record_err:
logger.error("Failed to finish execution record for job %s: %s", job["id"], record_err)
- if not isinstance(e, Exception):
+ if not contained:
raise
return False
finally:
diff --git a/gateway/platforms/api_server.py b/gateway/platforms/api_server.py
index 88e4fd9e41d..90b020e8c5b 100644
--- a/gateway/platforms/api_server.py
+++ b/gateway/platforms/api_server.py
@@ -236,8 +236,7 @@ def _clean_request_string(value: Any) -> Optional[str]:
# model_options decoding lives in the topical sibling (line-cap offset);
# re-exported here so importers are unaffected.
-from gateway.platforms.api_server_request_options import (
- _request_reasoning_config, _request_service_tier)
+from gateway.platforms.api_server_request_options import _request_reasoning_config, _request_service_tier
def _apply_runtime_agent_overrides(
@@ -3964,9 +3963,10 @@ class APIServerAdapter(OpenAICompatRoutesMixin, BasePlatformAdapter):
retryable=False)
return False
try:
- mws = [mw for mw in (
- self._make_profile_prefix_middleware(), cors_middleware, body_limit_middleware,
- security_headers_middleware) if mw is not None]
+ from gateway.platforms.api_server_config_guard import config_unavailable_middleware
+ mws = [mw for mw in ( # the config guard outermost: entering a profile scope reads config
+ config_unavailable_middleware, self._make_profile_prefix_middleware(), cors_middleware,
+ body_limit_middleware, security_headers_middleware) if mw is not None]
self._app = web.Application(middlewares=mws, client_max_size=MAX_REQUEST_BYTES)
assert self._app is not None
# Native routes + multiplex /p/<profile>/ mirrors (the prefix middleware validates and
diff --git a/gateway/platforms/api_server_config_guard.py b/gateway/platforms/api_server_config_guard.py
new file mode 100644
index 00000000000..9a526f4344c
--- /dev/null
+++ b/gateway/platforms/api_server_config_guard.py
@@ -0,0 +1,23 @@
+"""aiohttp middleware answering a config-backend outage instead of letting it end the gateway.
+
+``ConfigBackendUnavailable`` is a ``SystemExit`` on purpose (config readers must not serve
+defaults instead of the selected backend's config). Escaping an aiohttp handler task, a
+``SystemExit`` stops the event loop, so a remote-config plane blip during one request took the
+whole gateway down. This middleware turns it into that request's 503.
+"""
+
+from __future__ import annotations
+
+from aiohttp import web
+
+from hermes_cli.config_backend import ConfigBackendUnavailable
+
+
+async def config_unavailable_middleware(request: web.Request, handler):
+ try:
+ return await handler(request)
+ except ConfigBackendUnavailable as exc:
+ return web.json_response(
+ {"error": {"message": str(exc.code), "type": "server_error", "code": "config_backend_unavailable"}},
+ status=503)
diff --git a/gateway/platforms/base.py b/gateway/platforms/base.py
index cb3d51dc76f..016a74b9e02 100644
--- a/gateway/platforms/base.py
+++ b/gateway/platforms/base.py
@@ -4577,8 +4577,8 @@ class BasePlatformAdapter(ABC):
await self._run_processing_hook("on_processing_complete", event, ProcessingOutcome.FAILURE)
logger.error("[%s] Error handling message: %s", self.name, e, exc_info=True)
_thread_metadata = (await self._notify_turn_error(event, e)) or _thread_metadata
- # SystemExit/KeyboardInterrupt propagate; other BaseExceptions are contained.
- if isinstance(e, (SystemExit, KeyboardInterrupt)):
+ from hermes_cli.config_backend import ConfigBackendUnavailable as _outage # this turn only
+ if isinstance(e, (SystemExit, KeyboardInterrupt)) and not isinstance(e, _outage): # rest contained
raise
finally:
await self._release_turn_marker(event)
diff --git a/gateway/readiness.py b/gateway/readiness.py
index 651aab21028..2b31cfa9ead 100644
--- a/gateway/readiness.py
+++ b/gateway/readiness.py
@@ -49,12 +49,15 @@ def _probe_state_db(home: Path) -> dict[str, Any]:
def _probe_config(home: Path) -> dict[str, Any]:
- from hermes_cli.config_backend import config_exists, read_config_doc_readonly
+ from hermes_cli.config_backend import ConfigBackendUnavailable, config_exists, read_config_doc_readonly
path = home / "config.yaml"
- if not config_exists(path):
- return _check("ok", "using defaults")
try:
+ if not config_exists(path):
+ return _check("ok", "using defaults")
raw = read_config_doc_readonly(path)
+ except ConfigBackendUnavailable:
+ # A SystemExit on purpose (readers must not serve defaults); this probe exists to report it.
+ return _check("degraded", "config backend unavailable")
except Exception as exc:
return _check("degraded", f"invalid config ({type(exc).__name__})")
return _check("ok") if raw is None or isinstance(raw, dict) else _check("degraded", "top level is not a mapping")
diff --git a/gateway/run_profile_reconcile.py b/gateway/run_profile_reconcile.py
index 59dd4cf7af5..db79c2a7d0a 100644
--- a/gateway/run_profile_reconcile.py
+++ b/gateway/run_profile_reconcile.py
@@ -27,6 +27,7 @@ from pathlib import Path
from typing import Any, Dict, Optional
from gateway.run_shutdown import _log_suppressed
+from hermes_cli.config_backend import ConfigBackendUnavailable
from utils import file_cache_version
logger = logging.getLogger(__name__)
@@ -120,7 +121,7 @@ class GatewayProfileReconcileMixin:
await self.reconcile_served_profiles(reason="watcher")
except asyncio.CancelledError:
raise
- except Exception:
+ except (Exception, ConfigBackendUnavailable): # a config outage is a SystemExit by design
logger.warning("Served-profile reconcile failed; retrying next cycle", exc_info=True)
async def _sync_hooks_on_consent_change(self) -> None:
@@ -409,7 +410,7 @@ class GatewayProfileReconcileMixin:
return None
try:
await serve_profile_runtime(self, name, home)
- except Exception as exc:
+ except (Exception, ConfigBackendUnavailable) as exc: # unservable now: parked, retried by rescan
if self.session_authorities.for_home(home) is not None:
# Retirement of the failed attempt missed its deadline: its writer may still use
# this home, so the reservation and authority stay; the profile is parked (no
diff --git a/hermes_cli/config_read_errors.py b/hermes_cli/config_read_errors.py
index 5bf73ed12ef..b1f31d21fbc 100644
--- a/hermes_cli/config_read_errors.py
+++ b/hermes_cli/config_read_errors.py
@@ -129,7 +129,7 @@ class FailedConfigRead(dict):
after one transient EMFILE/EIO replaced the whole file with the fallback plus the caller's edit.
A dict subclass so the refusal survives the load→mutate→save round trip at every call site."""
- def __init__(self, data: Any = (), *, error: Exception):
+ def __init__(self, data: Any = (), *, error: BaseException):
super().__init__(data)
self.read_error = error
@@ -146,6 +146,8 @@ def _refuse_failed_read(config_path: Path, data: Any) -> None:
exc = data.read_error
if isinstance(exc, OSError):
raise _refuse_overwrite(config_path, "could not be read", exc, _read_error_fix(exc))
+ if isinstance(exc, SystemExit): # ConfigBackendUnavailable: the config backend could not serve it
+ raise RuntimeError(f"The config could not be read, so this change was not saved: {exc.code}")
raise _refuse_overwrite(
config_path, "has a formatting error", exc, _FIX_YAML.format(backups=_backups_dir_display()))
diff --git a/tests/cron/test_cron_provider_pin.py b/tests/cron/test_cron_provider_pin.py
index 153f908c686..59090885d40 100644
--- a/tests/cron/test_cron_provider_pin.py
+++ b/tests/cron/test_cron_provider_pin.py
@@ -173,6 +173,8 @@ class TestPinnedLocksTheMainModel:
from cron import jobs
(tmp_path / "config.yaml").write_text(f"model:\n default: {main_model}\n")
monkeypatch.setattr(jobs, "get_hermes_home", lambda: tmp_path, raising=True)
+ from cron import jobs_model_pin # the pin reads the main model's config where it lives
+ monkeypatch.setattr(jobs_model_pin, "get_hermes_home", lambda: tmp_path, raising=True)
state = {"jobs": []}
monkeypatch.setattr(jobs, "load_jobs", lambda: list(state["jobs"]), raising=True)
monkeypatch.setattr(jobs, "save_jobs", lambda j: state.__setitem__("jobs", list(j)), raising=True)
diff --git a/tests/plugins/config_backends/test_remote_unavailable_consumers.py b/tests/plugins/config_backends/test_remote_unavailable_consumers.py
new file mode 100644
index 00000000000..cef3ba69949
--- /dev/null
+++ b/tests/plugins/config_backends/test_remote_unavailable_consumers.py
@@ -0,0 +1,240 @@
+"""andrexibiza P1-B: ``ConfigBackendUnavailable`` is a ``SystemExit`` on purpose (a fail-open reader
+must never serve defaults instead of the selected backend's config; boot fails closed), so an
+``except Exception`` / ``suppress(Exception)`` around a config read does not catch it. Each
+long-lived consumer below handles it explicitly with ITS outcome, and its process keeps running.
+
+Every test raises the real ``ConfigBackendUnavailable`` from the config-read seam the consumer
+actually calls and asserts the contained outcome. Each one fails (the exit propagates) with its
+fix reverted.
+"""
+from __future__ import annotations
+
+import asyncio
+import threading
+from pathlib import Path
+
+import pytest
+
+from hermes_cli.config_backend import ConfigBackendUnavailable
+
+OUTAGE = "Remote Config: cannot load the config for profile 'default' from https://plane: HTTP 503"
+
+
+def _outage(*_a, **_kw):
+ raise ConfigBackendUnavailable(OUTAGE)
+
+
+# --- the six named sites ------------------------------------------------------------------------
+
+def test_readiness_probe_reports_config_backend_unavailable(tmp_path, monkeypatch):
+ """The probe whose job is to report config health reports the outage, never crashes."""
+ from gateway import readiness
+ monkeypatch.setattr("hermes_cli.config_backend.config_exists", _outage)
+ assert readiness._probe_config(tmp_path) == {"status": "degraded", "detail": "config backend unavailable"}
+
+ monkeypatch.setattr("hermes_cli.config_backend.config_exists", lambda _p: True)
+ monkeypatch.setattr("hermes_cli.config_backend.read_config_doc_readonly", _outage)
+ assert readiness._probe_config(tmp_path)["status"] == "degraded"
+
+ monkeypatch.setenv("HERMES_HOME", str(tmp_path))
+ result = readiness.collect_runtime_readiness(configured_model="m", runtime_status={})
+ assert result["status"] == "degraded" and result["checks"]["config"]["status"] == "degraded"
+
+
+def test_cron_tick_fails_only_that_fire_and_never_runs_on_defaults(tmp_path, monkeypatch):
+ """``_load_cron_job_config`` keeps refusing defaults (the SystemExit passes its ``except
+ Exception``); ``run_job`` — the fire's owner — turns it into this fire's failure result."""
+ import run_agent # run_job imports it; its import-time dotenv load reads config
+ from cron import scheduler
+ monkeypatch.setattr("hermes_cli.config_backend.config_exists", _outage)
+ with pytest.raises(ConfigBackendUnavailable):
+ scheduler._load_cron_job_config({"id": "j", "model": "m"}, "j", "job")
+
+ # Through the real run_job as the fire's owner: the pre-agent gates pass, then the read refuses.
+ monkeypatch.setattr("gateway.session_cron.current_execution", lambda: (None, "s", "j", "fire"))
+ monkeypatch.setattr(scheduler, "_prepare_job_prompt", lambda *a, **k: (None, "do the thing"))
+ monkeypatch.setattr(scheduler, "_reload_dotenv_and_publish_delivery_target", lambda job: None)
+ monkeypatch.setattr(scheduler, "_get_hermes_home", lambda: tmp_path)
+ success, doc, final, error = scheduler.run_job({"id": "j", "name": "job", "prompt": "p"},
+ execution_id="fire")
+ assert success is False and final == ""
+ assert error and "ConfigBackendUnavailable" in error and "HTTP 503" in error
+ assert "FAILED" in doc
+
+
+def test_cron_job_body_records_the_outage_and_keeps_the_ticker(monkeypatch, tmp_path):
+ """The body owner (``_run_one_job_body``) records a fire that escaped with the outage as failed
+ and returns, instead of re-raising it into the ticker thread like a real SystemExit."""
+ from cron import executions, jobs, scheduler
+ monkeypatch.setattr(scheduler, "run_job", lambda job, **kw: _outage())
+ marked = []
+ real_mark = scheduler.mark_job_run
+ monkeypatch.setattr(scheduler, "mark_job_run", lambda *a, **kw: (marked.append((a, kw)), real_mark(*a, **kw)))
+ with jobs.use_cron_store(tmp_path / "cron"):
+ job = jobs.create_job(prompt="report", schedule="every 1h", deliver="local")
+ fire = executions.create_execution(job["id"], source="test")["id"]
+ assert scheduler._run_one_job_body(dict(jobs.get_job(job["id"]), execution_id=fire)) is False
+ assert executions.get_execution(fire)["status"] == "failed"
+ assert marked and marked[0][0][1] is False and "HTTP 503" in marked[0][0][2]
+ assert jobs.get_job(job["id"])["last_status"] == "error"
+
+
+def test_cron_pin_refuses_instead_of_dropping_the_lock(monkeypatch, tmp_path):
+ """``pinned=True`` reads the main model; with the plane down that is an invalid request
+ (ValueError, the create/update path's refusal), never ``(None, None)`` = silently unpinned
+ and never a SystemExit out of the cronjob tool or RPC."""
+ from cron import jobs
+ from cron.jobs_model_pin import _main_model_pin
+ monkeypatch.setattr("hermes_cli.config_backend.config_exists", _outage)
+ with pytest.raises(ValueError, match="cannot pin the job to the main model.*HTTP 503"):
+ _main_model_pin()
+ with jobs.use_cron_store(tmp_path / "cron"):
+ with pytest.raises(ValueError, match="cannot pin"):
+ jobs.create_job(prompt="report", schedule="every 1h", deliver="local", pinned=True)
+ assert jobs.load_jobs() == []
+
+
+def test_tui_profile_cwd_returns_none(monkeypatch, tmp_path):
+ from tui_gateway import server
+ monkeypatch.setattr(server, "config_exists", _outage)
+ assert server._profile_configured_cwd(tmp_path) is None
+ assert server._default_session_cwd() # the session-create tail still answers a cwd
+
+
+def test_tui_raw_config_read_is_a_failed_read_that_refuses_to_save(monkeypatch, tmp_path):
+ """``_load_cfg_raw`` answers its designed degraded value (``FailedConfigRead``) and the
+ write-back primitive refuses to persist it, so the RPC answers an error instead of dying
+ or saving a ``{}`` over the plane's document."""
+ from hermes_cli.config_read_errors import FailedConfigRead, _refuse_failed_read
+ from tui_gateway import server
+ server._drop_cfg_cache()
+ monkeypatch.setattr(server, "config_version", _outage)
+ monkeypatch.setattr(server, "config_exists", lambda _p: True)
+ data = server._load_cfg_raw()
+ assert isinstance(data, FailedConfigRead) and data == {}
+ assert isinstance(data.read_error, ConfigBackendUnavailable)
+ with pytest.raises(RuntimeError) as refused:
+ _refuse_failed_read(tmp_path / "config.yaml", data)
+ assert "could not be read" in str(refused.value) and "HTTP 503" in str(refused.value)
+ assert "formatting error" not in str(refused.value) # not a YAML problem; nothing to edit
+
+
+def test_prompt_attachments_degrade_to_the_profile_home_store(monkeypatch, tmp_path):
+ """``file.attach`` keeps working: storage falls back to ``""`` (the profile-home store the
+ sandbox mounts and the vision allowlist always cover)."""
+ from tui_gateway import server
+ monkeypatch.setattr("hermes_cli.config_backend.config_exists", _outage)
+ assert server._profile_attachments_storage(tmp_path) == ""
+ session = {"profile_home": str(tmp_path), "cwd": str(tmp_path)}
+ assert server._session_home_dir(session, "attachments") == tmp_path / "attachments"
+
+
+# --- the owners above those sites (the rest of the class) --------------------------------------
+
+def test_tui_pooled_rpc_answers_once_with_the_outage(monkeypatch):
+ """The request owner: a long handler that hits the outage still writes one error frame."""
+ from tui_gateway import server
+ monkeypatch.setitem(server._methods, "cc.refused", lambda rid, params: _outage())
+ monkeypatch.setattr(server, "_LONG_HANDLERS", server._LONG_HANDLERS | {"cc.refused"})
+ written = threading.Event()
+ frames = []
+
+ class Recorder:
+ def write(self, obj):
+ frames.append(obj)
+ written.set()
+ return True
+
+ def close(self):
+ pass
+
+ assert server.dispatch({"jsonrpc": "2.0", "id": "r1", "method": "cc.refused", "params": {}}, Recorder()) is None
+ assert written.wait(10)
+ assert frames[0]["id"] == "r1" and "HTTP 503" in frames[0]["error"]["message"]
+
+
+def test_tui_agent_build_thread_reports_the_outage(monkeypatch):
+ """The deferred agent-build thread records ``agent_error`` and releases its waiters (a
+ SystemExit would have ended the thread with ``agent_ready`` never set)."""
+ from tui_gateway import server
+ sid = "build-outage"
+ session = {"session_key": "k", "agent": None, "agent_ready": threading.Event(),
+ "history_lock": threading.Lock(), "profile_home": None}
+ monkeypatch.setitem(server._sessions, sid, session)
+ monkeypatch.setattr(server, "_await_resume_history", lambda sid, current: True)
+ monkeypatch.setattr(server, "_bind_build_profile_scopes", lambda home: None)
+ monkeypatch.setattr(server, "_make_agent", lambda *a, **k: _outage())
+ monkeypatch.setattr(server, "_deferred_build_agent_kwargs", lambda current, db: {})
+ monkeypatch.setattr(server, "_emit", lambda *a, **k: None)
+ monkeypatch.setattr("tui_gateway.entry.ensure_mcp_discovery_started", lambda: None, raising=False)
+ server._start_agent_build(sid, session)
+ session["_agent_build_thread"].join(10)
+ assert session["agent_ready"].is_set()
+ assert "HTTP 503" in str(session.get("agent_error"))
+
+
+def test_gateway_turn_owner_contains_the_outage_and_notifies():
+ """A platform turn that hits the outage notifies the user and ends that turn only: re-raised,
+ a SystemExit out of the background task stops the gateway's event loop."""
+ from tests.gateway.test_baseexception_turn_notify import _event, _make_adapter, _raising_handler
+ from gateway.session import build_session_key
+ adapter = _make_adapter(_raising_handler(ConfigBackendUnavailable(OUTAGE)))
+ event = _event()
+ asyncio.run(adapter._process_message_background(event, build_session_key(event.source)))
+ assert adapter.sent and "error" in adapter.sent[0].lower()
+
+
+def test_gateway_api_server_answers_503():
+ pytest.importorskip("aiohttp")
+ from aiohttp import web
+ from aiohttp.test_utils import TestClient, TestServer
+ from gateway.platforms.api_server_config_guard import config_unavailable_middleware
+
+ async def handler(_request):
+ _outage()
+
+ async def main():
+ app = web.Application(middlewares=[config_unavailable_middleware])
+ app.router.add_get("/v1/x", handler)
+ async with TestClient(TestServer(app)) as client:
+ resp = await client.get("/v1/x")
+ return resp.status, await resp.json()
+
+ status, body = asyncio.run(main())
+ assert status == 503 and body["error"]["code"] == "config_backend_unavailable"
+
+
+def test_gateway_profile_reconcile_parks_the_profile(monkeypatch, tmp_path):
+ """Serving a new multiplexed profile whose first config fetch fails parks that profile (the
+ rescan retries it) instead of ending the reconcile watcher and the loop."""
+ from gateway import run_profile_reconcile as rpr
+ released = []
+
+ class Runner(rpr.GatewayProfileReconcileMixin):
+ session_authorities = type("A", (), {"for_home": staticmethod(lambda home: None)})()
+
+ async def serve(runner, name, home):
+ _outage()
+
+ monkeypatch.setattr("gateway.run_runtime.reserve_profile_home", lambda *a: None)
+ monkeypatch.setattr("gateway.run_runtime.release_profile_home", lambda runner, home: released.append(home))
+ monkeypatch.setattr("gateway.run_runtime.serve_profile_runtime", serve)
+ verdict = asyncio.run(Runner()._serve_profile_runtime("work", Path(tmp_path)))
+ assert verdict and "HTTP 503" in verdict and released == [Path(tmp_path)]
+
+
+def test_compute_host_turn_answers_turn_error(monkeypatch):
+ """The dashboard compute host's turn worker answers ``turn.error`` and keeps serving."""
+ import io
+ import json as _json
+ from tui_gateway.compute_host import ComputeHost
+ out = io.StringIO()
+ host = ComputeHost(stdout=out, heartbeat_secs=0)
+ monkeypatch.setattr(host, "_ensure_server_session", lambda server, frame: _outage())
+ try:
+ host._run_real_turn({"sid": "s1", "request_id": "q1", "text": "hi"})
+ finally:
+ host.close()
+ frames = [_json.loads(line) for line in out.getvalue().splitlines()]
+ assert [f["type"] for f in frames] == ["turn.error"]
+ assert frames[0]["request_id"] == "q1" and "HTTP 503" in frames[0]["message"]
diff --git a/tui_gateway/compute_host.py b/tui_gateway/compute_host.py
index 78a609a3be7..9c8b398a025 100644
--- a/tui_gateway/compute_host.py
+++ b/tui_gateway/compute_host.py
@@ -24,6 +24,7 @@ import uuid
from pathlib import Path
from typing import Any, Callable, Collection
+from hermes_cli.config_backend import ConfigBackendUnavailable
from tui_gateway.host_supervisor import MUTATOR_ROUTE_TABLE, _build_sha
@@ -186,7 +187,7 @@ class ComputeHost:
try:
from tui_gateway import server
body(server, sid, request_id)
- except Exception as exc:
+ except (Exception, ConfigBackendUnavailable) as exc: # health: allow BLE001 -- the frame's one reply (the outage is a SystemExit)
if on_error is not None:
on_error(sid)
self._reply(error_kind, sid, request_id, **error_extra, message=str(exc))
@@ -291,7 +292,7 @@ class ComputeHost:
self._reply(
"turn.end", sid, request_id, **meta, interrupted=interrupted, ended_ns=now_ns(),
session_info=session_info, session_info_emitted=True)
- except Exception as exc:
+ except (Exception, ConfigBackendUnavailable) as exc: # health: allow BLE001 -- the turn's one reply (the outage is a SystemExit)
with contextlib.suppress(Exception):
from tui_gateway import server
session = server._sessions.get(sid)
diff --git a/tui_gateway/prompt_attachments.py b/tui_gateway/prompt_attachments.py
index d8b8e4e7b0e..b43006ae5e8 100644
--- a/tui_gateway/prompt_attachments.py
+++ b/tui_gateway/prompt_attachments.py
@@ -128,17 +128,25 @@ def _profile_attachments_storage(profile_home) -> str:
Read from THAT profile's config.yaml — ``file.attach`` runs before ``prompt.submit``
installs the profile scope, so the process config still belongs to the launch profile
- (same reason as ``_profile_configured_cwd``)."""
+ (same reason as ``_profile_configured_cwd``).
+
+ Best-effort, including when the config backend cannot serve the profile: ``""`` keeps the
+ profile-home store, which the sandbox mounts and the vision allowlist always cover, so the
+ attachment still reaches the agent (by an absolute instead of a workspace-relative ref)."""
import contextlib as _contextlib
+ from hermes_cli.config_backend import ConfigBackendUnavailable
home = Path(profile_home) if profile_home else _hermes_home
- with _contextlib.suppress(Exception):
- from hermes_cli.config_backend import config_exists
- from hermes_cli.config_effective import load_user_config_effective
- cfg_path = home / "config.yaml"
- if config_exists(cfg_path):
- attachments_cfg = load_user_config_effective(cfg_path).get("attachments")
- if isinstance(attachments_cfg, dict):
- return str(attachments_cfg.get("storage") or "").strip().lower()
+ try:
+ with _contextlib.suppress(Exception):
+ from hermes_cli.config_backend import config_exists
+ from hermes_cli.config_effective import load_user_config_effective
+ cfg_path = home / "config.yaml"
+ if config_exists(cfg_path):
+ attachments_cfg = load_user_config_effective(cfg_path).get("attachments")
+ if isinstance(attachments_cfg, dict):
+ return str(attachments_cfg.get("storage") or "").strip().lower()
+ except ConfigBackendUnavailable as exc:
+ logger.warning("attachments.storage for %s unavailable (%s); staging in the profile home", home, exc.code)
return ""
diff --git a/tui_gateway/server.py b/tui_gateway/server.py
index 9f45a54d44f..01f364ebbc7 100644
--- a/tui_gateway/server.py
+++ b/tui_gateway/server.py
@@ -28,7 +28,7 @@ from hermes_constants import (
get_hermes_home, get_hermes_home_override, get_process_hermes_home, profile_name_for_home,
reset_hermes_home_override, set_hermes_home_override)
from hermes_cli import config_backend as _config_backend
-from hermes_cli.config_backend import Changes, config_exists, supports_file_tooling
+from hermes_cli.config_backend import Changes, ConfigBackendUnavailable, config_exists, supports_file_tooling
from hermes_cli.env_loader import load_hermes_dotenv
from tui_gateway.pet_sheet import _clone_pet_payload, _pet_row_frame_counts, _pet_sheet_revision, _pet_state_rows
from utils import is_truthy_value
@@ -676,10 +676,13 @@ def _profile_configured_cwd(profile_home: Path | None) -> str | None:
"""
if profile_home is None:
return None
- with contextlib.suppress(Exception):
- from hermes_cli.config_effective import load_user_config_effective
- p = Path(profile_home) / "config.yaml"
- return _configured_cwd_from_cfg(load_user_config_effective(p)) if config_exists(p) else None
+ try:
+ with contextlib.suppress(Exception):
+ from hermes_cli.config_effective import load_user_config_effective
+ p = Path(profile_home) / "config.yaml"
+ return _configured_cwd_from_cfg(load_user_config_effective(p)) if config_exists(p) else None
+ except ConfigBackendUnavailable as exc: # a SystemExit: the agent build reports it; the cwd is a hint
+ logger.warning("terminal.cwd of %s unavailable: %s", profile_home, exc.code)
return None
@@ -1013,7 +1016,7 @@ def _load_cfg_raw() -> dict:
if _cfg_cache is not None and _cfg_sig == sig and _cfg_path == p:
return copy.deepcopy(_cfg_cache)
data = read_user_config_raw(p) if config_exists(p) else {}
- except Exception as exc:
+ except (Exception, ConfigBackendUnavailable) as exc: # health: allow BLE001 -- served as a FailedConfigRead, refused by _save_cfg
return FailedConfigRead(error=exc) # readable as {}, refused by _save_cfg
with _cfg_lock: # cache the RAW config: _save_cfg writes _cfg_cache back to disk
_cfg_cache, _cfg_sig, _cfg_path = copy.deepcopy(data), sig, p
diff --git a/tui_gateway/session_registry.py b/tui_gateway/session_registry.py
index 4d96df74c31..c85cd7831ae 100644
--- a/tui_gateway/session_registry.py
+++ b/tui_gateway/session_registry.py
@@ -281,7 +281,7 @@ def _start_agent_build(sid: str, session: dict) -> None:
# MCP fleet, and live-transport sessions are never reaped, so fleets would accumulate.
notify_registered = _wire_session_agent(sid, key, agent)
_announce_built_agent(sid, key, current, agent)
- except Exception as e:
+ except (Exception, ConfigBackendUnavailable) as e: # health: allow BLE001 -- reported as agent_error + error event
from agent.auxiliary_unavailable import ProviderNotConfiguredError
current["agent_error"] = str(e)
# A client can route "no provider is set up" to its setup flow instead of a dead-end
@@ -617,7 +617,7 @@ def _schedule_resume_hydration(sid: str, stored_id: str, db, *, close_db: bool =
{"message_count": session["resume_message_count"], "phase": "history", "status": "complete"})
_maybe_schedule_auto_continue(sid, session, stored_id)
_start_agent_build(sid, session)
- except Exception as exc:
+ except (Exception, ConfigBackendUnavailable) as exc: # health: allow BLE001 -- reported as the resume failure
if _sessions.get(sid) is not session:
return
message = resume_failed_message(exc)