Skip to content

The agent that grows with you

fix(telemetry): the pre-os._exit flush drains the root profile's host-count store too

Teknium0107da5d2a36
diff --git a/hermes_cli/observability/relay_shared_metrics.py b/hermes_cli/observability/relay_shared_metrics.py
index 96ffc260caf..a44190b124d 100644
--- a/hermes_cli/observability/relay_shared_metrics.py
+++ b/hermes_cli/observability/relay_shared_metrics.py
@@ -10,7 +10,7 @@ import threading
import uuid
from collections import OrderedDict, deque
from dataclasses import dataclass, field
-from time import monotonic, monotonic_ns
+from time import monotonic_ns
from typing import Any, Callable, Iterator
from agent import relay_runtime
@@ -1518,18 +1518,6 @@ def shutdown_runtimes() -> None:
runtime._safe(runtime.shutdown)
-def drain_deferred_writes(deadline: float) -> None:
- """Write each live store's deferred rows (its atexit ``drain_deferred``, which ``os._exit`` also
- skips), waiting on a busy store no later than ``deadline`` (``time.monotonic``)."""
- with _RUNTIME_LOCK:
- stores = [runtime.subscriber.store for runtime in _RUNTIMES.values() if isinstance(runtime, _Runtime)]
- for store in stores:
- remaining_ms = int((deadline - monotonic()) * 1000)
- if remaining_ms <= 0:
- return
- _Runtime._safe(store.drain_deferred, busy_timeout_ms=remaining_ms)
-
-
def close_session_run(session_id: str) -> None:
"""Emit the summary of a session a surface retired without a finalize hook (gateway
auto-reset, idle expiry). Never creates a runtime; unknown ids are a no-op."""
diff --git a/hermes_cli/observability/shared_metrics.py b/hermes_cli/observability/shared_metrics.py
index 956b98bbb38..5e8988f3146 100644
--- a/hermes_cli/observability/shared_metrics.py
+++ b/hermes_cli/observability/shared_metrics.py
@@ -7,6 +7,7 @@ import json
import logging
import sqlite3
import threading
+import time
import uuid
from collections.abc import Callable, Iterator
from contextlib import contextmanager
@@ -155,6 +156,25 @@ _SEND_COLUMNS = (
logger = logging.getLogger(__name__)
+# Every store in this process that ever deferred a write, the per-profile runtimes' stores and the
+# ones no runtime owns alike (the root profile's host-level active-profiles accumulator, the Desktop
+# store). ``os._exit`` skips each store's atexit drain, so a hard exit drains this set instead.
+_DEFERRING_STORES: list[SharedMetricsStore] = []
+_DEFERRING_STORES_LOCK = threading.Lock()
+
+
+def drain_deferring_stores(deadline: float) -> None:
+ """Write every store's deferred rows before an ``os._exit``, waiting on a busy store no later
+ than ``deadline`` (``time.monotonic``). Each deferred write runs against the store that deferred
+ it, so one profile's rows never land in another's database."""
+ with _DEFERRING_STORES_LOCK:
+ stores = list(_DEFERRING_STORES)
+ for store in stores:
+ remaining_ms = int((deadline - time.monotonic()) * 1000)
+ if remaining_ms <= 0:
+ return
+ store.drain_deferred(busy_timeout_ms=remaining_ms)
+
def _utc_now() -> datetime:
return datetime.now(UTC)
@@ -429,6 +449,8 @@ class SharedMetricsStore:
register, self._drain_at_exit = not self._drain_at_exit, True
if register:
atexit.register(self.drain_deferred)
+ with _DEFERRING_STORES_LOCK:
+ _DEFERRING_STORES.append(self)
if not busy:
raise
diff --git a/hermes_cli/observability/shared_metrics_process.py b/hermes_cli/observability/shared_metrics_process.py
index 98e2c6702c0..f82ba05af96 100644
--- a/hermes_cli/observability/shared_metrics_process.py
+++ b/hermes_cli/observability/shared_metrics_process.py
@@ -68,21 +68,25 @@ def stamp_exit(exit_kind: str, crash: str = "none") -> None:
def flush_before_hard_exit(timeout_s: float = EXIT_FLUSH_TIMEOUT_S) -> bool:
"""For a graceful stop about to ``os._exit``: run the shared-metrics exit flush its skipped atexit
- hooks would have run (close each live runtime's sessions under its own profile, write queued and
- deferred rows to the local store).
+ hooks would have run (close each live runtime's sessions under its own profile, write queued rows,
+ and write every store's deferred rows, the root profile's host-level store included, each into
+ the store that deferred it).
Bounded by ``timeout_s`` on a daemon thread, because the Relay flush waits on every managed call
still in flight and a wedged one must not hold the exit. Never imports the runtime module:
collection off (or never started) means nothing to flush and nothing created. Never raises; True
when the flush finished in time."""
runtimes = sys.modules.get("hermes_cli.observability.relay_shared_metrics")
- if runtimes is None:
+ stores = sys.modules.get("hermes_cli.observability.shared_metrics")
+ if runtimes is None and stores is None:
return True
deadline = time.monotonic() + max(0.0, timeout_s)
def _flush() -> None:
- runtimes.shutdown_runtimes()
- runtimes.drain_deferred_writes(deadline - _DRAIN_MARGIN_S) # done before the join gives up
+ if runtimes is not None:
+ runtimes.shutdown_runtimes()
+ if stores is not None: # every store that deferred a write, root/Desktop ones included
+ stores.drain_deferring_stores(deadline - _DRAIN_MARGIN_S) # done before the join gives up
try:
from agent.memory_provider import spawn_context_thread
diff --git a/tests/hermes_cli/test_shared_metrics_exit_flush.py b/tests/hermes_cli/test_shared_metrics_exit_flush.py
index de0362be091..172a49d7660 100644
--- a/tests/hermes_cli/test_shared_metrics_exit_flush.py
+++ b/tests/hermes_cli/test_shared_metrics_exit_flush.py
@@ -271,3 +271,45 @@ def test_a_wedged_exit_flush_is_bounded_and_runs_after_the_clean_exit_stamp(metr
assert finished_in_time == [False] # the barrier really was wedged
assert phase_at_flush == ["exited"]
assert time.monotonic() - started < 5
+
+
+def _engagement_state(home: Path) -> dict | None:
+ with sqlite3.connect(_store(home).as_uri() + "?mode=ro", uri=True) as conn:
+ row = conn.execute("SELECT value FROM telemetry_state WHERE key = 'engagement_day_state'").fetchone()
+ return json.loads(row[0]) if row else None
+
+
+def test_graceful_exit_drains_the_root_profiles_deferred_host_count(tmp_path, monkeypatch):
+ """The host-level active-profiles accumulator lives in the ROOT profile's store, a store no
+ per-profile runtime owns. When another process held it past the busy timeout, the profile's visit
+ was deferred in memory; the graceful exit still writes it there (and only there) once the store
+ frees, instead of losing it to ``os._exit``."""
+ import gateway.run as gateway_run
+ from hermes_cli.observability import shared_metrics_engagement as engagement
+ from hermes_cli.observability.shared_metrics import SharedMetricsStore
+
+ root = (tmp_path / "root").resolve()
+ profile = root / "profiles" / "work"
+ profile.mkdir(parents=True)
+ _write_policy(root, enabled=True, send=False)
+ monkeypatch.setenv("HERMES_HOME", str(profile))
+ resource = {"architecture": "x86_64", "hermes_version": "0.0.0", "install_method": "git", "os_family": "linux"}
+ profile_store = SharedMetricsStore(_store(profile), _store(profile).parent / "outbox")
+ engagement._root_store(root) # the store exists; another process now holds its write lock
+ holder = sqlite3.connect(_store(root), isolation_level=None)
+ holder.execute("BEGIN IMMEDIATE")
+ try:
+ engagement.record(profile_store, resource, surface="cli", route=None)
+ finally:
+ holder.execute("ROLLBACK")
+ holder.close()
+ assert _engagement_state(root) is None # deferred, not written
+ profile_state = _engagement_state(profile)
+
+ monkeypatch.setattr(gateway_run.os, "_exit", _fake_exit)
+ with pytest.raises(_Exited):
+ gateway_run._exit_after_graceful_shutdown(0)
+
+ root_state = wait_until(lambda: _engagement_state(root), what="the root's deferred host count")
+ assert root_state["owner"] is True and len(root_state["profiles"]) == 1
+ assert _engagement_state(profile) == profile_state # the profile's own store never got the root's row