The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(telemetry): the pre-os._exit flush drains the root profile's host-count store too
Teknium0107da5d2a36
diff --git a/hermes_cli/observability/relay_shared_metrics.py b/hermes_cli/observability/relay_shared_metrics.pyindex 96ffc260caf..a44190b124d 100644--- a/hermes_cli/observability/relay_shared_metrics.py+++ b/hermes_cli/observability/relay_shared_metrics.py@@ -10,7 +10,7 @@ import threadingimport uuidfrom collections import OrderedDict, dequefrom dataclasses import dataclass, field-from time import monotonic, monotonic_ns+from time import monotonic_nsfrom typing import Any, Callable, Iteratorfrom agent import relay_runtime@@ -1518,18 +1518,6 @@ def shutdown_runtimes() -> None:runtime._safe(runtime.shutdown)-def drain_deferred_writes(deadline: float) -> None:- """Write each live store's deferred rows (its atexit ``drain_deferred``, which ``os._exit`` also- skips), waiting on a busy store no later than ``deadline`` (``time.monotonic``)."""- with _RUNTIME_LOCK:- stores = [runtime.subscriber.store for runtime in _RUNTIMES.values() if isinstance(runtime, _Runtime)]- for store in stores:- remaining_ms = int((deadline - monotonic()) * 1000)- if remaining_ms <= 0:- return- _Runtime._safe(store.drain_deferred, busy_timeout_ms=remaining_ms)--def close_session_run(session_id: str) -> None:"""Emit the summary of a session a surface retired without a finalize hook (gatewayauto-reset, idle expiry). Never creates a runtime; unknown ids are a no-op."""diff --git a/hermes_cli/observability/shared_metrics.py b/hermes_cli/observability/shared_metrics.pyindex 956b98bbb38..5e8988f3146 100644--- a/hermes_cli/observability/shared_metrics.py+++ b/hermes_cli/observability/shared_metrics.py@@ -7,6 +7,7 @@ import jsonimport loggingimport sqlite3import threading+import timeimport uuidfrom collections.abc import Callable, Iteratorfrom contextlib import contextmanager@@ -155,6 +156,25 @@ _SEND_COLUMNS = (logger = logging.getLogger(__name__)+# Every store in this process that ever deferred a write, the per-profile runtimes' stores and the+# ones no runtime owns alike (the root profile's host-level active-profiles accumulator, the Desktop+# store). ``os._exit`` skips each store's atexit drain, so a hard exit drains this set instead.+_DEFERRING_STORES: list[SharedMetricsStore] = []+_DEFERRING_STORES_LOCK = threading.Lock()+++def drain_deferring_stores(deadline: float) -> None:+ """Write every store's deferred rows before an ``os._exit``, waiting on a busy store no later+ than ``deadline`` (``time.monotonic``). Each deferred write runs against the store that deferred+ it, so one profile's rows never land in another's database."""+ with _DEFERRING_STORES_LOCK:+ stores = list(_DEFERRING_STORES)+ for store in stores:+ remaining_ms = int((deadline - time.monotonic()) * 1000)+ if remaining_ms <= 0:+ return+ store.drain_deferred(busy_timeout_ms=remaining_ms)+def _utc_now() -> datetime:return datetime.now(UTC)@@ -429,6 +449,8 @@ class SharedMetricsStore:register, self._drain_at_exit = not self._drain_at_exit, Trueif register:atexit.register(self.drain_deferred)+ with _DEFERRING_STORES_LOCK:+ _DEFERRING_STORES.append(self)if not busy:raisediff --git a/hermes_cli/observability/shared_metrics_process.py b/hermes_cli/observability/shared_metrics_process.pyindex 98e2c6702c0..f82ba05af96 100644--- a/hermes_cli/observability/shared_metrics_process.py+++ b/hermes_cli/observability/shared_metrics_process.py@@ -68,21 +68,25 @@ def stamp_exit(exit_kind: str, crash: str = "none") -> None:def flush_before_hard_exit(timeout_s: float = EXIT_FLUSH_TIMEOUT_S) -> bool:"""For a graceful stop about to ``os._exit``: run the shared-metrics exit flush its skipped atexit- hooks would have run (close each live runtime's sessions under its own profile, write queued and- deferred rows to the local store).+ hooks would have run (close each live runtime's sessions under its own profile, write queued rows,+ and write every store's deferred rows, the root profile's host-level store included, each into+ the store that deferred it).Bounded by ``timeout_s`` on a daemon thread, because the Relay flush waits on every managed callstill in flight and a wedged one must not hold the exit. Never imports the runtime module:collection off (or never started) means nothing to flush and nothing created. Never raises; Truewhen the flush finished in time."""runtimes = sys.modules.get("hermes_cli.observability.relay_shared_metrics")- if runtimes is None:+ stores = sys.modules.get("hermes_cli.observability.shared_metrics")+ if runtimes is None and stores is None:return Truedeadline = time.monotonic() + max(0.0, timeout_s)def _flush() -> None:- runtimes.shutdown_runtimes()- runtimes.drain_deferred_writes(deadline - _DRAIN_MARGIN_S) # done before the join gives up+ if runtimes is not None:+ runtimes.shutdown_runtimes()+ if stores is not None: # every store that deferred a write, root/Desktop ones included+ stores.drain_deferring_stores(deadline - _DRAIN_MARGIN_S) # done before the join gives uptry:from agent.memory_provider import spawn_context_threaddiff --git a/tests/hermes_cli/test_shared_metrics_exit_flush.py b/tests/hermes_cli/test_shared_metrics_exit_flush.pyindex de0362be091..172a49d7660 100644--- a/tests/hermes_cli/test_shared_metrics_exit_flush.py+++ b/tests/hermes_cli/test_shared_metrics_exit_flush.py@@ -271,3 +271,45 @@ def test_a_wedged_exit_flush_is_bounded_and_runs_after_the_clean_exit_stamp(metrassert finished_in_time == [False] # the barrier really was wedgedassert phase_at_flush == ["exited"]assert time.monotonic() - started < 5+++def _engagement_state(home: Path) -> dict | None:+ with sqlite3.connect(_store(home).as_uri() + "?mode=ro", uri=True) as conn:+ row = conn.execute("SELECT value FROM telemetry_state WHERE key = 'engagement_day_state'").fetchone()+ return json.loads(row[0]) if row else None+++def test_graceful_exit_drains_the_root_profiles_deferred_host_count(tmp_path, monkeypatch):+ """The host-level active-profiles accumulator lives in the ROOT profile's store, a store no+ per-profile runtime owns. When another process held it past the busy timeout, the profile's visit+ was deferred in memory; the graceful exit still writes it there (and only there) once the store+ frees, instead of losing it to ``os._exit``."""+ import gateway.run as gateway_run+ from hermes_cli.observability import shared_metrics_engagement as engagement+ from hermes_cli.observability.shared_metrics import SharedMetricsStore++ root = (tmp_path / "root").resolve()+ profile = root / "profiles" / "work"+ profile.mkdir(parents=True)+ _write_policy(root, enabled=True, send=False)+ monkeypatch.setenv("HERMES_HOME", str(profile))+ resource = {"architecture": "x86_64", "hermes_version": "0.0.0", "install_method": "git", "os_family": "linux"}+ profile_store = SharedMetricsStore(_store(profile), _store(profile).parent / "outbox")+ engagement._root_store(root) # the store exists; another process now holds its write lock+ holder = sqlite3.connect(_store(root), isolation_level=None)+ holder.execute("BEGIN IMMEDIATE")+ try:+ engagement.record(profile_store, resource, surface="cli", route=None)+ finally:+ holder.execute("ROLLBACK")+ holder.close()+ assert _engagement_state(root) is None # deferred, not written+ profile_state = _engagement_state(profile)++ monkeypatch.setattr(gateway_run.os, "_exit", _fake_exit)+ with pytest.raises(_Exited):+ gateway_run._exit_after_graceful_shutdown(0)++ root_state = wait_until(lambda: _engagement_state(root), what="the root's deferred host count")+ assert root_state["owner"] is True and len(root_state["profiles"]) == 1+ assert _engagement_state(profile) == profile_state # the profile's own store never got the root's row