The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(config-remote): review round 9 blockers: read-based save diff, masked inherited route keys, honest editor acks, UTF-16 bootstrap parse
Ben Barclayb1bbb51e212d
diff --git a/hermes_cli/config.py b/hermes_cli/config.pyindex 3dcaffe7e89..48027c52471 100644--- a/hermes_cli/config.py+++ b/hermes_cli/config.py@@ -2420,7 +2420,8 @@ def save_config(# fail-closed read is the single authority here: ``read_raw_config()`` is cached and# swallows transient stat/open errors into ``{}``, and a ``{}`` at this point makes the# strip pass drop every user section whose value matches a default (#113301).- _raw_for_paths = require_readable_config_before_write(config_path)+ from hermes_cli.config_backend import keeping_read_base+ _raw_for_paths = keeping_read_base(lambda: require_readable_config_before_write(config_path))from hermes_cli.config_worker_save import rebase_for_save # frozen-worker writes + merge_existingconfig = rebase_for_save(config, _raw_for_paths, merge_existing)diff --git a/hermes_cli/config_backend.py b/hermes_cli/config_backend.pyindex 63bc39dac52..3af5c5d0b98 100644--- a/hermes_cli/config_backend.py+++ b/hermes_cli/config_backend.py@@ -19,6 +19,7 @@ whatever user layer the backend returns.from __future__ import annotationsimport os+import sysimport threadingfrom dataclasses import dataclass, fieldfrom pathlib import Path@@ -427,6 +428,27 @@ def write_config_keys(config_path: PathLike, updates: Mapping[str, Any]) -> Noneunset=tuple(k for k, v in updates.items() if v is None)))+def keeping_read_base(read):+ """Run ``read()`` (a pre-write guard read) without making it the read a later document save+ is diffed against: the caller's document came from ITS earlier read (Remote Config). A+ context that never read keeps the guard read. No-op unless the remote backend is loaded."""+ mod = sys.modules.get("plugins.config_backends.remote.backend")+ held = mod._READ_BASE.get() if mod is not None else None+ try:+ return read()+ finally:+ if held is not None:+ mod._READ_BASE.set(held)+++def write_changes(config_path: PathLike, changes: Changes) -> None:+ """One :class:`Changes` write. Unlike :func:`write_config_keys`, a ``None`` in ``set`` stores+ an explicit null (Remote Config: "no value", masking an inherited scalar); the file backend+ writes it as YAML null."""+ backend, target, is_layer = _route(config_path)+ backend.write_changes(target, changes) if is_layer else backend.write_path(target, changes)++def deployment_env_names() -> frozenset:"""The selected backend's deployment env names (which backend, which plane, which instance).diff --git a/hermes_cli/env_loader.py b/hermes_cli/env_loader.pyindex c9c6afbf0d5..92c02ca8fac 100644--- a/hermes_cli/env_loader.py+++ b/hermes_cli/env_loader.py@@ -450,6 +450,36 @@ def _load_dotenv_with_fallback(_sanitize_loaded_credentials() # httpx encodes headers as ASCII+def _dotenv_assignments(path: Path) -> list:+ """``(name, raw value)`` pairs of one dotenv file as the loader will see it once+ ``_sanitize_env_file_if_needed`` has run: :func:`_parse_dotenv_file` plus the sanitizer's UTF-16+ rewrite and NUL strip, for the pre-boot bootstrap reads that run before the sanitizer."""+ raw = path.read_bytes()+ if raw.startswith(codecs.BOM_UTF16_LE) or raw.startswith(codecs.BOM_UTF16_BE):+ # Notepad "Unicode": ``_sanitize_env_file_if_needed`` rewrites it as UTF-8 before the loader+ # reads it, so parse it as that rewrite will -- bootstrap / stage2 run before the sanitizer+ # and must select the same backend the runtime then boots.+ try:+ with io.TextIOWrapper(io.BytesIO(raw), encoding="utf-16", newline=None) as f:+ text = f.read()+ except UnicodeDecodeError:+ text = raw.decode("latin-1")+ else:+ try:+ # utf-8-sig strips a leading BOM (PowerShell 5.1 / Notepad); plain utf-8 would keep U+FEFF on the+ # first key name and silently drop it from os.environ under its canonical name.+ text = raw.decode("utf-8-sig")+ except UnicodeDecodeError:+ raw = raw.removeprefix(codecs.BOM_UTF8)+ text = raw.decode("latin-1")+ text = text.replace("\x00", "") # the sanitizer strips NULs too (BOM-less UTF-16 / NUL-padded ASCII)+ # Imported here, not at module level: gateway tests stub ``sys.modules["dotenv"]`` with a bare module+ # exposing only ``load_dotenv``, and ``gateway.run`` imports this module at import time.+ from dotenv.main import DotEnv++ return list(DotEnv(dotenv_path=None, stream=io.StringIO(text), interpolate=False).parse())++def _bootstrap_env(home: Path | None = None, project_env: Path | None = None) -> dict:"""The process env as ``load_hermes_dotenv`` leaves it for the names its dotenv layers define:the home's ``.env`` (override), ``.op.env`` (fills gaps, only without a process@@ -475,7 +505,7 @@ def _bootstrap_env(home: Path | None = None, project_env: Path | None = None) ->layers.append((managed_dir / ".env", True))for path, override in layers:try:- assignments = _parse_dotenv_file(path)+ assignments = _dotenv_assignments(path)except OSError:continueresolved = _resolve_dotenv_assignments(assignments, env, override=override)diff --git a/hermes_cli/model_switch.py b/hermes_cli/model_switch.pyindex bbcc69c61cf..85fccf3d0a9 100644--- a/hermes_cli/model_switch.py+++ b/hermes_cli/model_switch.py@@ -1867,8 +1867,16 @@ def persist_model_selection(result: ModelSwitchResult, config_path: Any = None)updates = model_selection_config_updates(result, planned_from)if not supports_file_tooling():# No config file (Remote Config): one keyed write, so a lock on any key refuses the whole- # switch instead of a document save silently omitting the locked part.- write_config_keys(path, {f"model.{key}": value for key, value in updates.items()})+ # switch instead of a document save silently omitting the locked part. The planned-from+ # block is the EFFECTIVE one, so a key it holds may come from an upper level, where a+ # profile-level unset cannot reach it (contract §5.4): an inherited old-endpoint api_key+ # would stay active at the new endpoint. A key the switch clears and the effective block+ # still holds is therefore masked with null ("no value"); an absent one is just unset.+ from hermes_cli.config_backend import Changes, write_changes+ held = planned_from if isinstance(planned_from, dict) else {}+ write_changes(path, Changes(+ set={f"model.{k}": v for k, v in updates.items() if v is not None or k in held},+ unset=tuple(f"model.{k}" for k, v in updates.items() if v is None and k not in held)))returnwith _CONFIG_LOCK:config = read_user_config_raw(path)diff --git a/plugins/config_backends/remote/backend.py b/plugins/config_backends/remote/backend.pyindex f10f69b0adb..fd7e8ffc029 100644--- a/plugins/config_backends/remote/backend.py+++ b/plugins/config_backends/remote/backend.py@@ -106,6 +106,28 @@ class _Private:_PRIVATE: contextvars.ContextVar[Optional[_Private]] = contextvars.ContextVar("remote_config_private", default=None)_PRIVATE_SERIAL = itertools.count(1)+# The diff base each profile had when THIS context last read it: {id(state): base}. A document+# save means "turn what I read into this", so its intent is diffed against that read, never+# against whatever a poll (or another writer) installed between the read and the save. Replaced,+# never mutated: copied contexts (threads, tasks) must not share one dict.+_READ_BASE: contextvars.ContextVar[Optional[dict]] = contextvars.ContextVar("remote_config_read_base", default=None)+++def _remember_read(st: _ProfileState) -> None:+ """Record (by reference: ``st.base`` / ``st.server_config`` are replaced, never mutated) the+ base this context just read under st.lock."""+ seen = dict(_READ_BASE.get() or {})+ seen[id(st)] = (st, st.base if st.base is not None else st.server_config)+ _READ_BASE.set(seen)+++def _read_base_for(st: _ProfileState) -> Optional[dict[str, Any]]:+ held = (_READ_BASE.get() or {}).get(id(st))+ if held is None or held[0] is not st or held[1] is None:+ return None # never read in this context, or a different (reset) profile state+ base = copy.deepcopy(held[1])+ base.pop("_config_version", None)+ return basedef _private_for(st: _ProfileState) -> Optional[_Private]:@@ -404,6 +426,8 @@ class RemoteBackend:def take():m = _private_for(st)+ if m is None:+ _remember_read(st)return (copy.deepcopy(m.doc if m is not None else st.doc), self._version_of(st),{encode(p): level for p, level in st.locks})@@ -416,6 +440,8 @@ class RemoteBackend:def take():m = _private_for(st)+ if m is None:+ _remember_read(st)return m.doc if m is not None else st.docreturn self._snapshot(st, take)@@ -433,7 +459,13 @@ class RemoteBackend:return (st.changed_ns, st.etag, st.gen)def version(self, home: Path) -> tuple[Any, ...]:- return self._version_of(self._state(home))+ # A config cache validates on this signature and then hands out the doc it cached for it:+ # that doc is what the caller now holds, so this is the read a later save diffs against.+ st = self._state(home)+ with st.lock:+ if _private_for(st) is None and st.postprocessed:+ _remember_read(st)+ return self._version_of(st)def exists(self, home: Path) -> bool:return True # the plane always resolves a document (an absent level is empty, contract R4)@@ -487,6 +519,7 @@ class RemoteBackend:raise ConfigWriteError(f"Remote Config write failed: {exc}", code="config_plane_unreachable") from excif resp.status == 200 and _effective_ok(resp.body):self._install(st, resp.body, resp.etag)+ _remember_read(st) # the caller now holds what it wroteself._warn_inherited(st, unsets)returnif resp.status == 409 and attempt == 1:@@ -507,9 +540,17 @@ class RemoteBackend:return basedef _intent(self, st: _ProfileState, changes: Changes) -> tuple[dict[KeyPath, Any], list[KeyPath]]:- """The key-level edit ``changes`` makes to the last read doc (§15.2 steps 1-5)."""+ """The key-level edit ``changes`` makes to the last read doc (§15.2 steps 1-5).++ A document is diffed against the base THIS context read it from (``_READ_BASE``): a poll+ landing between the caller's read and its save must not turn every key the poll changed+ into an edit of ours, sent with the already-current expectedVersion (no 409 to catch it).+ A keyed set/unset names its own keys, so it applies to the current base."""base = self._diff_base(st)if changes.document is not None:+ read = _read_base_for(st) if _private_for(st) is None else None+ if read is not None:+ base = readnew = to_wire(copy.deepcopy(changes.document))if not isinstance(new, dict):raise ConfigValueError("the config document must be a mapping")diff --git a/tests/plugins/config_backends/test_remote_backend.py b/tests/plugins/config_backends/test_remote_backend.pyindex c594c7bf9d6..6fc60932fae 100644--- a/tests/plugins/config_backends/test_remote_backend.py+++ b/tests/plugins/config_backends/test_remote_backend.py@@ -1654,3 +1654,50 @@ def test_tui_shared_metrics_set_refuses_a_locked_consent(plane, monkeypatch):assert answer["error"]["code"] == 4002 and "locked" in answer["error"]["message"]assert plane.patches() == [] and recorded == []+++def test_poll_between_read_and_save_sends_only_the_callers_edit(plane, monkeypatch):+ """A poll landing after ``set_config_value`` read the doc and before it saved must not turn+ the keys that poll changed into edits of ours (sent with the current expectedVersion, so no+ 409 could catch them): only the requested key is sent; the poll's changes survive."""+ from hermes_cli import config as cfg+ prof = plane.profile("default")+ prof.update(values={"display": {"personality": "concise", "compact": False}}, version=1)+ backend = get_config_backend()+ orig = cfg._write_user_config++ def poll_before_save(path, doc):+ p = plane.profile("default")+ p["values"]["display"]["compact"] = True+ p["values"]["terminal"] = {"timeout": 321}+ p["version"] += 1+ assert backend.poll_one(backend._state(plane.home))+ return orig(path, doc)++ monkeypatch.setattr(cfg, "_write_user_config", poll_before_save)+ cfg.set_config_value("display.personality", "pirate")+ bodies = [r["body"] for r in plane.patches()]+ assert len(bodies) == 1, bodies+ assert bodies[0]["set"] == {"display.personality": "pirate"} and not bodies[0].get("unset"), bodies+ assert plane.profile("default")["values"] == {+ "display": {"personality": "pirate", "compact": True}, "terminal": {"timeout": 321}}+++def test_model_switch_masks_inherited_endpoint_credentials(plane, monkeypatch):+ """A switch to a new endpoint clears endpoint-bound keys. When an upper level supplies them,+ a profile-level unset cannot reach them, so the switch stores null ("no value") instead: the+ old endpoint's key never reaches the new endpoint."""+ from hermes_cli.config import read_user_config_raw+ from hermes_cli.model_switch import ModelSwitchResult, persist_model_selection+ plane.upper = {"model": {"default": "old", "provider": "custom:old", "base_url": "https://old.example/v1",+ "api_key": "${OLD_KEY}", "key_env": "OLD_KEY", "api_key_env": "OLD_KEY",+ "context_length": 123456}}+ persist_model_selection(ModelSwitchResult(+ success=True, new_model="new", target_provider="custom", provider_changed=True,+ base_url="https://new.example/v1", api_mode="chat_completions", is_global=True))+ body = plane.patches()[-1]["body"]+ for key in ("api_key", "key_env", "api_key_env", "context_length"):+ assert body["set"].get(f"model.{key}", "absent") is None, body+ model = read_user_config_raw()["model"]+ assert model["base_url"] == "https://new.example/v1"+ assert all(model.get(k) is None for k in ("api_key", "key_env", "api_key_env", "context_length")), modeldiff --git a/tests/plugins/config_backends/test_remote_grouped_and_migration_reads.py b/tests/plugins/config_backends/test_remote_grouped_and_migration_reads.pyindex 4e8702182cc..5e898d5f59b 100644--- a/tests/plugins/config_backends/test_remote_grouped_and_migration_reads.py+++ b/tests/plugins/config_backends/test_remote_grouped_and_migration_reads.py@@ -312,3 +312,47 @@ def test_migration_recursion_still_sees_its_private_copy(plane, monkeypatch):assert seen == [latest - 1]assert doc["new_key"] == 2 and "old_key" not in docassert plane.patches() == []+++def test_profile_editor_reports_a_locked_skills_edit_as_not_applied(plane, monkeypatch):+ """``profiles.configure`` saves its sections through bulk ``save_config``, which omits a+ Remote Config lock instead of raising. The editor must report that section unapplied, not+ acknowledge an edit the plane never stored; unlocked, the same edit lands and is reported."""+ import json+ from hermes_cli.config import read_user_config_raw+ plane.upper = {"skills": {"disabled": []}}+ plane.upper_locks = [{"path": "skills.disabled", "level": "tenant"}]+ server = _tui_server(monkeypatch, plane.home)++ class Transport:+ def __init__(self):+ self.frames, self.done = [], threading.Event()++ def write(self, obj):+ self.frames.append(obj)+ self.done.set()+ return True++ def close(self):+ pass++ def configure(rid):+ t = Transport()+ ret = server.dispatch({"jsonrpc": "2.0", "id": rid, "method": "profiles.configure",+ "params": {"name": "default", "disabled_skills": ["example-skill"]}}, t)+ if ret is not None:+ t.write(ret)+ assert t.done.wait(10), "no terminal response"+ return json.loads(json.dumps(t.frames[-1]))["result"]++ locked = configure(1)+ assert locked["applied"]["skills"] is False and locked["ok"] is False, locked+ assert plane.patches() == []+ assert read_user_config_raw()["skills"]["disabled"] == []++ plane.upper_locks = []+ backend = get_config_backend()+ backend.poll_one(backend._state(plane.home))+ unlocked = configure(2)+ assert unlocked["applied"]["skills"] is True and unlocked["ok"] is True, unlocked+ assert read_user_config_raw()["skills"]["disabled"] == ["example-skill"]diff --git a/tests/tools/test_stage2_hook_remote_config_skip.py b/tests/tools/test_stage2_hook_remote_config_skip.pyindex 30715a0ab5f..b15e29a0906 100644--- a/tests/tools/test_stage2_hook_remote_config_skip.py+++ b/tests/tools/test_stage2_hook_remote_config_skip.py@@ -123,3 +123,16 @@ def test_remote_from_managed_dotenv_skips(home):(home.parent / "managed" / ".env").write_text("HERMES_CONFIG_BACKEND=remote\n")assert _runtime_selects(home, None) == "remote"assert "MIGRATE_RAN" not in _run(home, None)++[email protected]("encode", [+ lambda t: t.encode("utf-16"), # Notepad "Unicode" (BOM)+ lambda t: t.encode("utf-16-le"), # BOM-less UTF-16: NUL-padded ASCII+], ids=["utf16-bom", "utf16-no-bom"])+def test_stage2_decides_like_the_runtime_for_utf16_dotenv(home, encode):+ """The runtime sanitizes a UTF-16 / NUL-padded .env before loading it; the early selection+ (stage2, the first config read) parses it the same way, so both pick remote."""+ (home / ".env").write_bytes(encode("OTHER=1\nHERMES_CONFIG_BACKEND=remote\n"))+ from hermes_cli.env_loader import selected_config_backend_name+ assert selected_config_backend_name(home) == "remote"+ assert "MIGRATE_RAN" not in _run(home, None)diff --git a/tui_gateway/methods_profiles.py b/tui_gateway/methods_profiles.pyindex 386f5c10702..22bc197973a 100644--- a/tui_gateway/methods_profiles.py+++ b/tui_gateway/methods_profiles.py@@ -738,15 +738,37 @@ def _configure_cfg_sections(profile_dir, params, applied) -> None:try:from hermes_cli.skills_config import save_disabled_skillssave_disabled_skills(cfg, _clean_names(params["disabled_skills"]))- applied["skills"] = True+ applied["skills"] = _landed(cfg, ("skills", "disabled"))cfg = load_config() or {}except Exception:applied["skills"] = Falseif isinstance(params.get("enabled_toolsets"), list):- applied["toolsets"] = _best_effort(lambda: _save_toolset_pin(cfg, params["enabled_toolsets"], save_config))+ applied["toolsets"] = _best_effort(lambda: _save_toolset_pin(cfg, params["enabled_toolsets"], save_config)) \+ and _landed(cfg, ("platform_toolsets", "cli"))if want_mcp:+ mcp_cfg = load_config() or {}applied["mcp_servers"] = _best_effort(lambda: _save_mcp_toggles(- load_config() or {}, params["enabled_mcp_servers"], launch_mcp, save_config))+ mcp_cfg, params["enabled_mcp_servers"], launch_mcp, save_config)) \+ and _landed(mcp_cfg, ("mcp_servers",), lambda v: {+ k: e.get("enabled") for k, e in v.items() if isinstance(e, dict)} if isinstance(v, dict) else v)+++def _landed(cfg: dict, path: tuple, key=lambda v: v) -> bool:+ """Whether the section a bulk ``save_config`` was asked to write now reads back as written.+ Under Remote Config a bulk save omits a locked key (with a stderr note) instead of raising, so+ an editor that reported success after the save would acknowledge an edit the plane refused.+ Compared on the user layer as read back (the effective doc in remote mode), absent == absent."""+ from hermes_cli.config import read_user_config_raw++ def at(doc):+ node = doc+ for seg in path:+ if not isinstance(node, dict) or seg not in node:+ return None+ node = node[seg]+ return node++ return key(at(read_user_config_raw() or {})) == key(at(cfg))def _unlink_asset_files(assets_dir, asset) -> int: