The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(telemetry): exit flush applies each profile's own send consent and collection gate
Tekniumb991bd203b63
diff --git a/hermes_cli/observability/relay_shared_metrics.py b/hermes_cli/observability/relay_shared_metrics.pyindex 5b810e40582..96ffc260caf 100644--- a/hermes_cli/observability/relay_shared_metrics.py+++ b/hermes_cli/observability/relay_shared_metrics.py@@ -10,10 +10,11 @@ import threadingimport uuidfrom collections import OrderedDict, dequefrom dataclasses import dataclass, field-from time import monotonic_ns-from typing import Any, Callable+from time import monotonic, monotonic_ns+from typing import Any, Callable, Iteratorfrom agent import relay_runtime+from hermes_constants import reset_hermes_home_override, set_hermes_home_overridefrom agent.portal_tags import get_conversation_contextfrom hermes_cli.version_info import get_version_info@@ -629,6 +630,16 @@ class _Runtime:_forget(self._sessions, session.session_id, session)def shutdown(self) -> None:+ # Callers (the atexit hook, an os._exit flush, a multiplex host's teardown) hold the launch+ # profile's scope: this profile's own collection gate and send consent decide its exit.+ with self._owner_scope():+ if not enabled(): # revoked since its last turn: drop the data, never export it+ if self._registered:+ self.deactivate()+ return+ self._close_and_export()++ def _close_and_export(self) -> None:with self._sessions_lock:self._active = Falsesession_ids = list(self._sessions)@@ -644,6 +655,15 @@ class _Runtime:self._deregister()self._release()+ @contextlib.contextmanager+ def _owner_scope(self) -> Iterator[None]:+ """Bind this runtime's profile home for work that can run outside its own turns."""+ token = set_hermes_home_override(self.host.profile_key)+ try:+ yield+ finally:+ reset_hermes_home_override(token)+def _deregister(self) -> None:self._safe(self.relay.subscribers.deregister, self._subscriber_name)self.host.release_managed_execution(self._subscriber_name)@@ -752,13 +772,15 @@ class _Runtime:self._flush_and_export(failure_message)def _flush_and_export(self, failure_message: str) -> None:- """Flush the Relay subscriber, then export; a failed flush skips the export."""+ """Flush the Relay subscriber, then export under this profile's scope (its send consent,+ whoever triggered the flush); a failed flush skips the export."""try:self.relay.subscribers.flush()except Exception:logger.warning(failure_message, exc_info=True)else:- self._export()+ with self._owner_scope():+ self._export()def _abort_session(self, session: _MetricsSession, base_event: dict[str, Any]) -> bool:"""Mark the session closing and system-abort its open tasks; False if already closing."""@@ -1273,9 +1295,11 @@ class _Runtime:with self._send_lock:# One in-flight pass per process; the next hook fire picks up what is pending.if self._send_thread is None or not self._send_thread.is_alive():- self._send_thread = threading.Thread(- target=self._run_send_pass, args=(resolved.endpoint,),- name="hermes-shared-metrics-send", daemon=True,+ from agent.memory_provider import spawn_context_thread++ # Context copy: the pass re-checks consent (still_consented) in THIS profile's scope.+ self._send_thread = spawn_context_thread(+ self._run_send_pass, name="hermes-shared-metrics-send", args=(resolved.endpoint,))self._send_thread.start()@@ -1487,13 +1511,25 @@ def start_task_run(def shutdown_runtimes() -> None:"""Run each live runtime's exit flush now, for a process that leaves through ``os._exit`` (the``hermes -z`` exit) and so never reaches the atexit hook that closes sessions and writes- ``task_run.finished`` / ``session.count``."""+ ``task_run.finished`` / ``session.count``. Each runtime binds its own profile while it does."""with _RUNTIME_LOCK:runtimes = [runtime for runtime in _RUNTIMES.values() if isinstance(runtime, _Runtime)]for runtime in runtimes:runtime._safe(runtime.shutdown)+def drain_deferred_writes(deadline: float) -> None:+ """Write each live store's deferred rows (its atexit ``drain_deferred``, which ``os._exit`` also+ skips), waiting on a busy store no later than ``deadline`` (``time.monotonic``)."""+ with _RUNTIME_LOCK:+ stores = [runtime.subscriber.store for runtime in _RUNTIMES.values() if isinstance(runtime, _Runtime)]+ for store in stores:+ remaining_ms = int((deadline - monotonic()) * 1000)+ if remaining_ms <= 0:+ return+ _Runtime._safe(store.drain_deferred, busy_timeout_ms=remaining_ms)++def close_session_run(session_id: str) -> None:"""Emit the summary of a session a surface retired without a finalize hook (gatewayauto-reset, idle expiry). Never creates a runtime; unknown ids are a no-op."""diff --git a/tests/hermes_cli/test_shared_metrics_exit_flush.py b/tests/hermes_cli/test_shared_metrics_exit_flush.pyindex 264304a0263..f8b0df6bd74 100644--- a/tests/hermes_cli/test_shared_metrics_exit_flush.py+++ b/tests/hermes_cli/test_shared_metrics_exit_flush.py@@ -3,11 +3,14 @@The gateway (SIGTERM, update restart, idle exit all funnel through ``_exit_after_graceful_shutdown``),the compute host, a SIGTERMed kanban worker and an orphaned slash worker leave via ``os._exit``, so asession still open at the stop never wrote its summary rows to the opt-in local store. The flush is-bounded and creates nothing when collection is off.+bounded, creates nothing when collection is off, and runs every profile's runtime under that profile's+own consent (a multiplex gateway exits from the launch profile's scope)."""from __future__ import annotations+import json+import osimport sqlite3import threadingimport time@@ -20,6 +23,8 @@ from hermes_cli import lifecycle, pluginsfrom hermes_cli.observability import relay_shared_metricsfrom hermes_cli.observability import shared_metrics_process as process_metricsfrom hermes_cli.plugins import PluginManager+from hermes_constants import reset_hermes_home_override, set_hermes_home_override+from tests.waiting import wait_untilclass _Exited(Exception):@@ -42,12 +47,29 @@ def _metric_names(home: Path) -> set[str]:return {name for (name,) in conn.execute("SELECT metric_name FROM counter_aggregates")}-def metrics_home(tmp_path, monkeypatch):- """Real NeMo Relay runtime over a temp HERMES_HOME; ``policy['on']`` is the opt-in answer."""+def _open_consent_windows(home: Path) -> list[tuple]:+ with sqlite3.connect(_store(home).as_uri() + "?mode=ro", uri=True) as conn:+ return conn.execute("SELECT opened_at FROM send_consent_windows WHERE closed_at IS NULL").fetchall()+++def _require_native_relay() -> None:relay = pytest.importorskip("nemo_relay")if getattr(relay, "_native", None) is None:pytest.skip("NeMo Relay native binding is unavailable on this platform")+++def _zero_plugin_runtimes(monkeypatch) -> None:+ relay_shared_metrics._reset_for_tests()+ relay_runtime._reset_for_tests()+ manager = PluginManager()+ manager._discovered = True # a runtime with zero plugins (#64178)+ monkeypatch.setattr(plugins, "_plugin_manager", manager)+++def metrics_home(tmp_path, monkeypatch):+ """Real NeMo Relay runtime over a temp HERMES_HOME; ``policy['on']`` is the opt-in answer."""+ _require_native_relay()home = tmp_path / "hermes-home"policy = {"on": True}monkeypatch.setenv("HERMES_HOME", str(home))@@ -55,30 +77,23 @@ def metrics_home(tmp_path, monkeypatch):"hermes_cli.config.read_raw_config_readonly",lambda: {"telemetry": {"shared_metrics": {"enabled": policy["on"], "send": False}}},)- relay_shared_metrics._reset_for_tests()- relay_runtime._reset_for_tests()- manager = PluginManager()- manager._discovered = True # a runtime with zero plugins (#64178)- monkeypatch.setattr(plugins, "_plugin_manager", manager)+ _zero_plugin_runtimes(monkeypatch)yield home, policyrelay_shared_metrics._reset_for_tests()relay_runtime._reset_for_tests()-def _finished_task_in_open_session() -> None:+def _finished_task_in_open_session(session_id: str = "open-session") -> None:"""One completed task in a session that stays open, as a messaging or Desktop chat does."""- event = {"session_id": "open-session", "task_id": "task-1", "platform": "cli"}+ event = {"session_id": session_id, "task_id": "task-1", "platform": "cli"}lifecycle.invoke_hook("on_session_start", **event, turn_id="turn-1")relay_shared_metrics.start_task_run(**event)lifecycle.invoke_hook("pre_llm_call", **event, turn_id="turn-1", messages=["hello"]) # a user turnrelay_shared_metrics.finish_task_run(**event, result={"completed": True, "final_response": "ok"})-def _wait_for(home: Path, metric: str, deadline_s: float = 10.0) -> None:- deadline = time.monotonic() + deadline_s- while metric not in _metric_names(home):- assert time.monotonic() < deadline, f"{metric} never reached the store"- time.sleep(0.05)+def _wait_for(home: Path, metric: str) -> None:+ wait_until(lambda: metric in _metric_names(home), what=f"{metric} in the store")def test_graceful_gateway_exit_writes_the_open_sessions_summary(metrics_home, monkeypatch):@@ -167,6 +182,51 @@ def test_graceful_exit_with_collection_off_creates_no_store(metrics_home, monkeyassert not (home / "telemetry").exists()+def _write_policy(home: Path, **policy) -> None:+ config = home / "config.yaml"+ config.write_text(json.dumps({"telemetry": {"shared_metrics": policy}}))+ later = time.time() + 5 # a rewrite inside one mtime tick must still read as a change+ os.utime(config, (later, later))+++def test_exit_flush_applies_each_profiles_own_consent(tmp_path, monkeypatch):+ """A multiplex gateway exits from the launch profile's scope. A routed profile with sending off+ keeps it off (no consent window opened on its store), and one that turned collection off after+ its turn gets no exit rows; both rulings are that profile's own, not the launch profile's."""+ import gateway.run as gateway_run++ _require_native_relay()+ launch, send_off, revoked = (tmp_path / name for name in ("launch", "send-off", "revoked"))+ for home in (launch, send_off, revoked):+ home.mkdir()+ _write_policy(launch, enabled=True, send=True, endpoint="http://127.0.0.1:9/never")+ _write_policy(send_off, enabled=True, send=False)+ _write_policy(revoked, enabled=True, send=False)+ monkeypatch.setenv("HERMES_HOME", str(launch))+ _zero_plugin_runtimes(monkeypatch)+ try:+ for home in (send_off, revoked): # routed turns, each under its own profile scope+ token = set_hermes_home_override(str(home))+ try:+ _finished_task_in_open_session(f"open-{home.name}")+ _wait_for(home, "hermes.task_run.finished")+ finally:+ reset_hermes_home_override(token)+ _write_policy(revoked, enabled=False, send=False)+ rows_before_exit = _metric_names(revoked)++ monkeypatch.setattr(gateway_run.os, "_exit", _fake_exit)+ with pytest.raises(_Exited):+ gateway_run._exit_after_graceful_shutdown(0)++ assert "hermes.session.count" in _metric_names(send_off) # its own consent still flushes it+ assert _open_consent_windows(send_off) == []+ assert _metric_names(revoked) == rows_before_exit+ finally:+ relay_shared_metrics._reset_for_tests()+ relay_runtime._reset_for_tests()++def test_a_wedged_exit_flush_never_holds_the_exit(monkeypatch):release = threading.Event()monkeypatch.setattr(relay_shared_metrics, "shutdown_runtimes", lambda: release.wait(30))diff --git a/website/docs/developer-guide/relay-shared-metrics.md b/website/docs/developer-guide/relay-shared-metrics.mdindex f24970833a5..046839e2f36 100644--- a/website/docs/developer-guide/relay-shared-metrics.md+++ b/website/docs/developer-guide/relay-shared-metrics.md@@ -365,7 +365,12 @@ like `interactive`, as they did before the value existed; `background` and `deleruns are unattended and excluded there. Packages written before `one_shot` existedcarry these runs as `interactive` and still validate. `hermes -z` leaves through`os._exit`, so it closes its metrics session before exiting rather than relying on the-atexit hook.+atexit hook. Graceful gateway, compute-host, SIGTERMed kanban-worker and orphaned+slash-worker stops also leave through `os._exit` and run the same flush, bounded at 3 s.+Every runtime's exit flush, export and send pass binds that runtime's own profile, so a+process serving several profiles applies each profile's own collection and send consent:+a profile that turned collection off since its last turn drops its in-memory rows+instead of writing them.Each tool invocation is represented by a Relay tool lifecycle named`hermes.tool_call`. The terminal counter contains only bounded tool category,