Skip to content

The agent that grows with you

fix(telemetry): exit flush applies each profile's own send consent and collection gate

Tekniumb991bd203b63
diff --git a/hermes_cli/observability/relay_shared_metrics.py b/hermes_cli/observability/relay_shared_metrics.py
index 5b810e40582..96ffc260caf 100644
--- a/hermes_cli/observability/relay_shared_metrics.py
+++ b/hermes_cli/observability/relay_shared_metrics.py
@@ -10,10 +10,11 @@ import threading
import uuid
from collections import OrderedDict, deque
from dataclasses import dataclass, field
-from time import monotonic_ns
-from typing import Any, Callable
+from time import monotonic, monotonic_ns
+from typing import Any, Callable, Iterator
from agent import relay_runtime
+from hermes_constants import reset_hermes_home_override, set_hermes_home_override
from agent.portal_tags import get_conversation_context
from hermes_cli.version_info import get_version_info
@@ -629,6 +630,16 @@ class _Runtime:
_forget(self._sessions, session.session_id, session)
def shutdown(self) -> None:
+ # Callers (the atexit hook, an os._exit flush, a multiplex host's teardown) hold the launch
+ # profile's scope: this profile's own collection gate and send consent decide its exit.
+ with self._owner_scope():
+ if not enabled(): # revoked since its last turn: drop the data, never export it
+ if self._registered:
+ self.deactivate()
+ return
+ self._close_and_export()
+
+ def _close_and_export(self) -> None:
with self._sessions_lock:
self._active = False
session_ids = list(self._sessions)
@@ -644,6 +655,15 @@ class _Runtime:
self._deregister()
self._release()
+ @contextlib.contextmanager
+ def _owner_scope(self) -> Iterator[None]:
+ """Bind this runtime's profile home for work that can run outside its own turns."""
+ token = set_hermes_home_override(self.host.profile_key)
+ try:
+ yield
+ finally:
+ reset_hermes_home_override(token)
+
def _deregister(self) -> None:
self._safe(self.relay.subscribers.deregister, self._subscriber_name)
self.host.release_managed_execution(self._subscriber_name)
@@ -752,13 +772,15 @@ class _Runtime:
self._flush_and_export(failure_message)
def _flush_and_export(self, failure_message: str) -> None:
- """Flush the Relay subscriber, then export; a failed flush skips the export."""
+ """Flush the Relay subscriber, then export under this profile's scope (its send consent,
+ whoever triggered the flush); a failed flush skips the export."""
try:
self.relay.subscribers.flush()
except Exception:
logger.warning(failure_message, exc_info=True)
else:
- self._export()
+ with self._owner_scope():
+ self._export()
def _abort_session(self, session: _MetricsSession, base_event: dict[str, Any]) -> bool:
"""Mark the session closing and system-abort its open tasks; False if already closing."""
@@ -1273,9 +1295,11 @@ class _Runtime:
with self._send_lock:
# One in-flight pass per process; the next hook fire picks up what is pending.
if self._send_thread is None or not self._send_thread.is_alive():
- self._send_thread = threading.Thread(
- target=self._run_send_pass, args=(resolved.endpoint,),
- name="hermes-shared-metrics-send", daemon=True,
+ from agent.memory_provider import spawn_context_thread
+
+ # Context copy: the pass re-checks consent (still_consented) in THIS profile's scope.
+ self._send_thread = spawn_context_thread(
+ self._run_send_pass, name="hermes-shared-metrics-send", args=(resolved.endpoint,)
)
self._send_thread.start()
@@ -1487,13 +1511,25 @@ def start_task_run(
def shutdown_runtimes() -> None:
"""Run each live runtime's exit flush now, for a process that leaves through ``os._exit`` (the
``hermes -z`` exit) and so never reaches the atexit hook that closes sessions and writes
- ``task_run.finished`` / ``session.count``."""
+ ``task_run.finished`` / ``session.count``. Each runtime binds its own profile while it does."""
with _RUNTIME_LOCK:
runtimes = [runtime for runtime in _RUNTIMES.values() if isinstance(runtime, _Runtime)]
for runtime in runtimes:
runtime._safe(runtime.shutdown)
+def drain_deferred_writes(deadline: float) -> None:
+ """Write each live store's deferred rows (its atexit ``drain_deferred``, which ``os._exit`` also
+ skips), waiting on a busy store no later than ``deadline`` (``time.monotonic``)."""
+ with _RUNTIME_LOCK:
+ stores = [runtime.subscriber.store for runtime in _RUNTIMES.values() if isinstance(runtime, _Runtime)]
+ for store in stores:
+ remaining_ms = int((deadline - monotonic()) * 1000)
+ if remaining_ms <= 0:
+ return
+ _Runtime._safe(store.drain_deferred, busy_timeout_ms=remaining_ms)
+
+
def close_session_run(session_id: str) -> None:
"""Emit the summary of a session a surface retired without a finalize hook (gateway
auto-reset, idle expiry). Never creates a runtime; unknown ids are a no-op."""
diff --git a/tests/hermes_cli/test_shared_metrics_exit_flush.py b/tests/hermes_cli/test_shared_metrics_exit_flush.py
index 264304a0263..f8b0df6bd74 100644
--- a/tests/hermes_cli/test_shared_metrics_exit_flush.py
+++ b/tests/hermes_cli/test_shared_metrics_exit_flush.py
@@ -3,11 +3,14 @@
The gateway (SIGTERM, update restart, idle exit all funnel through ``_exit_after_graceful_shutdown``),
the compute host, a SIGTERMed kanban worker and an orphaned slash worker leave via ``os._exit``, so a
session still open at the stop never wrote its summary rows to the opt-in local store. The flush is
-bounded and creates nothing when collection is off.
+bounded, creates nothing when collection is off, and runs every profile's runtime under that profile's
+own consent (a multiplex gateway exits from the launch profile's scope).
"""
from __future__ import annotations
+import json
+import os
import sqlite3
import threading
import time
@@ -20,6 +23,8 @@ from hermes_cli import lifecycle, plugins
from hermes_cli.observability import relay_shared_metrics
from hermes_cli.observability import shared_metrics_process as process_metrics
from hermes_cli.plugins import PluginManager
+from hermes_constants import reset_hermes_home_override, set_hermes_home_override
+from tests.waiting import wait_until
class _Exited(Exception):
@@ -42,12 +47,29 @@ def _metric_names(home: Path) -> set[str]:
return {name for (name,) in conn.execute("SELECT metric_name FROM counter_aggregates")}
-def metrics_home(tmp_path, monkeypatch):
- """Real NeMo Relay runtime over a temp HERMES_HOME; ``policy['on']`` is the opt-in answer."""
+def _open_consent_windows(home: Path) -> list[tuple]:
+ with sqlite3.connect(_store(home).as_uri() + "?mode=ro", uri=True) as conn:
+ return conn.execute("SELECT opened_at FROM send_consent_windows WHERE closed_at IS NULL").fetchall()
+
+
+def _require_native_relay() -> None:
relay = pytest.importorskip("nemo_relay")
if getattr(relay, "_native", None) is None:
pytest.skip("NeMo Relay native binding is unavailable on this platform")
+
+
+def _zero_plugin_runtimes(monkeypatch) -> None:
+ relay_shared_metrics._reset_for_tests()
+ relay_runtime._reset_for_tests()
+ manager = PluginManager()
+ manager._discovered = True # a runtime with zero plugins (#64178)
+ monkeypatch.setattr(plugins, "_plugin_manager", manager)
+
+
+def metrics_home(tmp_path, monkeypatch):
+ """Real NeMo Relay runtime over a temp HERMES_HOME; ``policy['on']`` is the opt-in answer."""
+ _require_native_relay()
home = tmp_path / "hermes-home"
policy = {"on": True}
monkeypatch.setenv("HERMES_HOME", str(home))
@@ -55,30 +77,23 @@ def metrics_home(tmp_path, monkeypatch):
"hermes_cli.config.read_raw_config_readonly",
lambda: {"telemetry": {"shared_metrics": {"enabled": policy["on"], "send": False}}},
)
- relay_shared_metrics._reset_for_tests()
- relay_runtime._reset_for_tests()
- manager = PluginManager()
- manager._discovered = True # a runtime with zero plugins (#64178)
- monkeypatch.setattr(plugins, "_plugin_manager", manager)
+ _zero_plugin_runtimes(monkeypatch)
yield home, policy
relay_shared_metrics._reset_for_tests()
relay_runtime._reset_for_tests()
-def _finished_task_in_open_session() -> None:
+def _finished_task_in_open_session(session_id: str = "open-session") -> None:
"""One completed task in a session that stays open, as a messaging or Desktop chat does."""
- event = {"session_id": "open-session", "task_id": "task-1", "platform": "cli"}
+ event = {"session_id": session_id, "task_id": "task-1", "platform": "cli"}
lifecycle.invoke_hook("on_session_start", **event, turn_id="turn-1")
relay_shared_metrics.start_task_run(**event)
lifecycle.invoke_hook("pre_llm_call", **event, turn_id="turn-1", messages=["hello"]) # a user turn
relay_shared_metrics.finish_task_run(**event, result={"completed": True, "final_response": "ok"})
-def _wait_for(home: Path, metric: str, deadline_s: float = 10.0) -> None:
- deadline = time.monotonic() + deadline_s
- while metric not in _metric_names(home):
- assert time.monotonic() < deadline, f"{metric} never reached the store"
- time.sleep(0.05)
+def _wait_for(home: Path, metric: str) -> None:
+ wait_until(lambda: metric in _metric_names(home), what=f"{metric} in the store")
def test_graceful_gateway_exit_writes_the_open_sessions_summary(metrics_home, monkeypatch):
@@ -167,6 +182,51 @@ def test_graceful_exit_with_collection_off_creates_no_store(metrics_home, monkey
assert not (home / "telemetry").exists()
+def _write_policy(home: Path, **policy) -> None:
+ config = home / "config.yaml"
+ config.write_text(json.dumps({"telemetry": {"shared_metrics": policy}}))
+ later = time.time() + 5 # a rewrite inside one mtime tick must still read as a change
+ os.utime(config, (later, later))
+
+
+def test_exit_flush_applies_each_profiles_own_consent(tmp_path, monkeypatch):
+ """A multiplex gateway exits from the launch profile's scope. A routed profile with sending off
+ keeps it off (no consent window opened on its store), and one that turned collection off after
+ its turn gets no exit rows; both rulings are that profile's own, not the launch profile's."""
+ import gateway.run as gateway_run
+
+ _require_native_relay()
+ launch, send_off, revoked = (tmp_path / name for name in ("launch", "send-off", "revoked"))
+ for home in (launch, send_off, revoked):
+ home.mkdir()
+ _write_policy(launch, enabled=True, send=True, endpoint="http://127.0.0.1:9/never")
+ _write_policy(send_off, enabled=True, send=False)
+ _write_policy(revoked, enabled=True, send=False)
+ monkeypatch.setenv("HERMES_HOME", str(launch))
+ _zero_plugin_runtimes(monkeypatch)
+ try:
+ for home in (send_off, revoked): # routed turns, each under its own profile scope
+ token = set_hermes_home_override(str(home))
+ try:
+ _finished_task_in_open_session(f"open-{home.name}")
+ _wait_for(home, "hermes.task_run.finished")
+ finally:
+ reset_hermes_home_override(token)
+ _write_policy(revoked, enabled=False, send=False)
+ rows_before_exit = _metric_names(revoked)
+
+ monkeypatch.setattr(gateway_run.os, "_exit", _fake_exit)
+ with pytest.raises(_Exited):
+ gateway_run._exit_after_graceful_shutdown(0)
+
+ assert "hermes.session.count" in _metric_names(send_off) # its own consent still flushes it
+ assert _open_consent_windows(send_off) == []
+ assert _metric_names(revoked) == rows_before_exit
+ finally:
+ relay_shared_metrics._reset_for_tests()
+ relay_runtime._reset_for_tests()
+
+
def test_a_wedged_exit_flush_never_holds_the_exit(monkeypatch):
release = threading.Event()
monkeypatch.setattr(relay_shared_metrics, "shutdown_runtimes", lambda: release.wait(30))
diff --git a/website/docs/developer-guide/relay-shared-metrics.md b/website/docs/developer-guide/relay-shared-metrics.md
index f24970833a5..046839e2f36 100644
--- a/website/docs/developer-guide/relay-shared-metrics.md
+++ b/website/docs/developer-guide/relay-shared-metrics.md
@@ -365,7 +365,12 @@ like `interactive`, as they did before the value existed; `background` and `dele
runs are unattended and excluded there. Packages written before `one_shot` existed
carry these runs as `interactive` and still validate. `hermes -z` leaves through
`os._exit`, so it closes its metrics session before exiting rather than relying on the
-atexit hook.
+atexit hook. Graceful gateway, compute-host, SIGTERMed kanban-worker and orphaned
+slash-worker stops also leave through `os._exit` and run the same flush, bounded at 3 s.
+Every runtime's exit flush, export and send pass binds that runtime's own profile, so a
+process serving several profiles applies each profile's own collection and send consent:
+a profile that turned collection off since its last turn drops its in-memory rows
+instead of writing them.
Each tool invocation is represented by a Relay tool lifecycle named
`hermes.tool_call`. The terminal counter contains only bounded tool category,