The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(telemetry): a busy message saved behind a pause or restart is not dropped; auto-resumed chats count toward their own profile's peak
Tekniuma41d1dfca08e
diff --git a/gateway/run_startup.py b/gateway/run_startup.pyindex 90983102ed4..a3194e52a2e 100644--- a/gateway/run_startup.py+++ b/gateway/run_startup.py@@ -27,6 +27,7 @@ from gateway.restart import (DEFAULT_GATEWAY_CRON_DRAIN_TIMEOUT, GATEWAY_FATAL_CONFIG_EXIT_CODE, is_global_startup_conflict)from gateway.run_shutdown import _log_suppressed, _send_error+from hermes_cli.observability.shared_metrics_busy import note_running_chat_ownerfrom gateway.shutdown_watchdog import (DEFAULT_HEARTBEAT_INTERVAL_S, DEFAULT_LOOP_WATCHDOG_INTERVAL_S,DEFAULT_LOOP_WATCHDOG_MAX_STRIKES, DEFAULT_LOOP_WATCHDOG_TIMEOUT_S, loop_heartbeat_forever,@@ -659,6 +660,7 @@ class GatewayStartupMixin:_resume_state = self._session_state(entry.session_key)_resume_state.turn.agent = _AGENT_PENDING_SENTINEL_resume_state.turn.started_ts = time.time()+ note_running_chat_owner(self, entry.session_key, source) # this loop runs in the launch scopeself._persist_active_agents()# Empty-text internal event: the _is_resume_pending branch prepends the reason-aware note.event = MessageEvent(text="", message_type=MessageType.TEXT, source=source, internal=True)diff --git a/gateway/session_ingress.py b/gateway/session_ingress.pyindex d86eafae7e7..3f5691dc06f 100644--- a/gateway/session_ingress.py+++ b/gateway/session_ingress.py@@ -8,7 +8,8 @@ from dataclasses import replacefrom gateway.platforms.event import MessageEvent, ProcessingOutcomefrom gateway.session_envelope import restore_nativefrom gateway.session_producer_lifecycle import close_producer_turn, producer_turn-from hermes_cli.observability.shared_metrics_busy import busy_turn_started, note_busy_admitted, settle_busy_arrival+from hermes_cli.observability.shared_metrics_busy import (+ busy_turn_started, note_busy_admitted, note_busy_parked, settle_busy_arrival)from hermes_state_runtime import RuntimeStoreErrorlogger = logging.getLogger(__name__)@@ -94,6 +95,8 @@ async def admit_message(authority, event):# The drain refused this head (``_refuse_head``): cancelled, not saved behind a pause.return (f'⚠️ Hermes did not answer this message ({exc.reason}): the conversation it was ''queued for moved, so it was dropped. Send it again to continue.')+ if row is not None:+ note_busy_parked(event) # still queued (saved): answered after the pause, not droppedreturn pause_notice(authority, receipt.ref, exc.reason)finally:if adopters.get(receipt.admission_id) is event:diff --git a/hermes_cli/observability/shared_metrics_busy.py b/hermes_cli/observability/shared_metrics_busy.pyindex f814792b405..b34132b4006 100644--- a/hermes_cli/observability/shared_metrics_busy.py+++ b/hermes_cli/observability/shared_metrics_busy.py@@ -56,7 +56,7 @@ def busy_arrival_fields(*, platform: Any, disposition: str, seconds: float | Non# ---- busy arrivals ----class _Arrival:- __slots__ = ("admission_id", "admitting", "arrived", "disposition", "done", "home", "merged", "platform")+ __slots__ = ("admission_id", "admitting", "arrived", "disposition", "done", "home", "merged", "parked", "platform")def __init__(self, platform: Any, home: str | None) -> None:self.arrived = time.monotonic()@@ -66,6 +66,7 @@ class _Arrival:self.admission_id: str | None = Noneself.merged: list[_Arrival] = []self.done = False+ self.parked = False # its row stays queued behind a paused FIFO: not dropped, owed later# FIFO admission id -> arrival whose row is still queued. Touched only on the gateway loop.@@ -137,10 +138,20 @@ def note_busy_admitted(event: Any, admission_id: str, status: str) -> None:_queued[tag.admission_id] = tagwhile len(_queued) > _QUEUED_MAX:_queued.popitem(last=False)[1].done = True- elif status in ("started", "terminal"):+ elif status in ("started", "unknown", "terminal"): # ``unknown``: it started, then its turn was lost_resolve(tag, tag.disposition, time.monotonic())+def note_busy_parked(event: Any) -> None:+ """The handler is answering with a pause notice (``runtime_draining``, ``session_busy``,+ ``unknown_execution``) while the message's row is still queued: it is saved, not dropped. It+ counts only if this process later runs that row (``busy_turn_started``); a restart owes it to+ the next process, which has no record of the arrival."""+ tag = _tag(event)+ if tag is not None:+ tag.parked = True++def busy_turn_started(admission_id: Any) -> None:"""The FIFO claimed this row and is running it: a queued busy arrival reached the agent."""tag = _queued.pop(str(admission_id), None)@@ -151,9 +162,9 @@ def busy_turn_started(admission_id: Any) -> None:def settle_busy_arrival(event: Any, *, discarded: bool = False) -> None:"""The busy handler for ``event`` returned (or its held burst was dropped by /stop, /new):an arrival that reached admission and never ran was dropped. One stopped by a restart is not- counted: its committed row outlives this process."""+ counted: its committed row outlives this process. A parked one stays owed (``note_busy_parked``)."""tag = _tag(event)- if tag is None:+ if tag is None or (tag.parked and not discarded):returnif tag.admission_id is not None:_queued.pop(tag.admission_id, None)@@ -212,6 +223,17 @@ def _home_key(home: Any) -> str:return _resolved[raw]+def note_running_chat_owner(runner: Any, session_key: str, source: Any) -> None:+ """``session_key`` is being claimed outside its profile's scope (the startup auto-resume loop):+ count it toward the profile that owns the session, resolved the way its turn will resolve it."""+ try:+ home = _home_key(runner._resolve_profile_home_for_source(source))+ with _peak_lock:+ _running[str(session_key)] = home+ except Exception:+ logger.debug("Running-chat owner not resolved", exc_info=True)++def observe_running_chats(session_keys: Any) -> None:"""The runner's running-turn set at a turn boundary (claim or release). A newly running chatbelongs to the profile whose scope claimed it."""diff --git a/tests/hermes_cli/test_shared_metrics_busy.py b/tests/hermes_cli/test_shared_metrics_busy.pyindex a29da75b4cd..f6648af6dd9 100644--- a/tests/hermes_cli/test_shared_metrics_busy.py+++ b/tests/hermes_cli/test_shared_metrics_busy.py@@ -188,3 +188,80 @@ def test_day_row_reports_each_profiles_peak_of_chats_running_at_once(tmp_path, massert contract.counter_dimensions_are_valid(contract.ENGAGEMENT_DAY_METRIC, row)legacy = {k: v for k, v in row.items() if k != contract.BUSY_PEAK_FIELD}assert contract.counter_dimensions_are_valid(contract.ENGAGEMENT_DAY_METRIC, legacy)+++class _PausingAuthority:+ """The admission surface ``admit_message`` reads: the row commits as ``queued``, then the FIFO+ pauses (a runtime stop releases every waiter with ``runtime_draining``, a worker on the chat+ with ``session_busy``) while the row stays queued for later."""++ def __init__(self):+ self.db, self.native_waiters, self.waiters = None, set(), {}+ self.sessions = {"sid": SimpleNamespace(pause_notified=False, source=None)}++ async def admit_native(self, event):+ return SimpleNamespace(status="queued", admission_id="adm-p", ref=SimpleNamespace(session_id="sid"))++[email protected]("reason", ["runtime_draining", "session_busy"])+def test_a_busy_message_saved_behind_a_pause_is_not_dropped(rows, monkeypatch, reason):+ """The user is told the message is saved and its FIFO row stays ``queued``: no ``dropped``+ row. When this process later runs that row (a ``session_busy`` pause lifting), it counts as+ ``queued`` with its wait; after a restart the next process runs it and nothing is counted."""+ import hermes_state_runtime+ from gateway.session_ingress import admit_message+ from hermes_state_runtime import RuntimeStoreError++ monkeypatch.setattr(hermes_state_runtime, "get_session_admission",+ lambda db, admission_id: {"status": "queued"})+ authority, replies = _PausingAuthority(), []++ async def runner_admits(event):+ busy.note_busy_admitting(event)+ replies.append(await admit_message(authority, event))++ async def scenario():+ adapter = _Adapter(runner_admits)+ adapter._busy_text_mode = "interrupt" # not a debounced burst: straight to the runner+ await dispatch_busy_burst(adapter, _message("b"), KEY)+ await asyncio.sleep(0)+ authority.waiters["adm-p"].set_exception(RuntimeStoreError(reason))+ await asyncio.gather(*adapter._background_tasks)++ asyncio.run(scenario())+ assert "saved" in replies[0]+ assert rows() == []+ busy.busy_turn_started("adm-p") # the pause lifted and this process ran the row+ assert [(d["disposition"], d["wait_bucket"]) for _m, d in rows()] == [("queued", "lt_5s")]+++def test_an_auto_resumed_chat_counts_toward_its_owning_profiles_peak(tmp_path, monkeypatch):+ """The startup auto-resume loop claims chats from the launch profile's scope; a resumed chat+ of another profile counts toward that profile's peak, not the launch profile's."""+ from gateway.session import SessionEntry+ from tests.gateway.restart_test_helpers import make_restart_runner, make_restart_source++ monkeypatch.setattr("gateway.run._write_runtime_status_quiet", lambda **_: None)+ monkeypatch.setattr(busy, "_running", {})+ monkeypatch.setattr(busy, "_peaks", {})+ monkeypatch.setattr(busy, "_now", lambda: DAY1)+ launch, beta = tmp_path / "launch", tmp_path / "beta"+ monkeypatch.setenv("HERMES_HOME", str(launch))+ runner, _adapter = make_restart_runner()+ runner.session_authorities = []+ runner._resolve_profile_home_for_source = lambda source: beta+ entry = SessionEntry(session_key="agent:beta:telegram:dm:c", session_id="s-beta",+ created_at=datetime.now(), updated_at=datetime.now(),+ origin=make_restart_source(chat_id="c"), platform=Platform.TELEGRAM, chat_type="dm",+ resume_pending=True, resume_reason="restart_interrupted",+ last_resume_marked_at=datetime.now())+ runner.session_store._entries = {entry.session_key: entry}++ async def resume():+ runner._run_startup_resume_event = lambda *a, **k: asyncio.sleep(0)+ assert runner._schedule_resume_pending_sessions() == 1+ await asyncio.sleep(0)++ asyncio.run(resume())+ day = busy._day(DAY1)+ assert (busy.day_peak(beta, day), busy.day_peak(launch, day)) == (1, 0)diff --git a/website/docs/developer-guide/relay-shared-metrics.md b/website/docs/developer-guide/relay-shared-metrics.mdindex a727deab73c..0a1d02a1791 100644--- a/website/docs/developer-guide/relay-shared-metrics.md+++ b/website/docs/developer-guide/relay-shared-metrics.md@@ -496,7 +496,7 @@ itself ships.| `hermes.update.run` | kind, outcome, failed_stage, duration_bucket, from_version_age_bucket, apply_mode | Whether updates succeed, how long they take, where they fail, and how stale the version being updated from was. `outcome` is `success`, `noop`, `failed`, `refused`, or `partial` (committed, then interrupted or left the user's stashed changes parked; `hermes update` rows only). A `partial` run counts as an attempt and as a failure in the per-user update rate (these runs read `failed` before, so the trend stays continuous), and its reasons sit with the runs that applied the code and then failed a follow-up. `hermes update` rows are derived from the final update receipt, once per run (`kind` is `desktop` when Desktop's source-checkout hand-off ran it); a run the pre-update interpreter finishes, or the bare bootstrap interpreter finishes after dependency preparation failed (it cannot load the config reader), is parked locally with only the bounded fields the row needs and counted by the next start, which purges it unrecorded when collection is off (nothing is parked when that interpreter can already tell collection is off); Desktop packaged self-updates (`apply_mode=package`) are reported once by the app, after the restart that applies them. A lock refusal (`lock_held`) of the update child a Desktop hand-off launched (the update marker names it, or its parent, as the hand-off's delegate) reads `failed`, not `refused`: the Desktop already quit to run that update and reports it as a failed one. That includes the checkout lock held by another mutator (a Desktop build, a venv sync), not only another update running. || `hermes.update.stage` | stage, outcome, duration_bucket | Per-stage result and wall time of `hermes update` (plan, snapshot, apply, deps, build, restart, verify), from the receipt's stage timestamps. || `hermes.process.exit` | process_kind, exit_kind, crash_class | How CLI / TUI / gateway / serve / cron-tick processes end (`clean`, `crash` with an exception family only, `killed`, `watchdog`), reported by the next start in the same profile from a local marker (a start with collection off deletes these markers, and pending `provider_setup` ones, unreported). Turns aborted by a turn watchdog also count as `exit_kind=watchdog`. |-| `hermes.gateway.busy_arrival.count` | platform, disposition (`queued`/`steered`/`interrupted`/`merged`/`dropped`/`other`), wait bucket (`lt_5s`, `5s_to_30s`, `30s_to_2m`, `2m_to_10m`, `gte_10m`, `never`) | How often people message an agent that is still busy in their chat, and how long that message waits before the agent sees it. One row per human message that arrives while the chat's turn is running, recorded once its fate is known: `queued` (ran later as its own turn), `steered` (delivered into the running turn: `busy_input_mode: steer`, or an interrupt-mode correction that redirects the turn), `interrupted` (stopped the running turn, then ran as the next one), `merged` (folded into another busy message of the same burst; its wait is until that message reached the agent), `dropped` (never ran: cleared by `/stop` or `/new`, refused, or over the queue cap). The wait runs from arrival to its own turn starting or the steer landing, `never` when dropped. Commands, approval answers, internal events and background-process notifications are not counted. A message still queued when the gateway restarts is not counted: its saved row runs in the next process. |+| `hermes.gateway.busy_arrival.count` | platform, disposition (`queued`/`steered`/`interrupted`/`merged`/`dropped`/`other`), wait bucket (`lt_5s`, `5s_to_30s`, `30s_to_2m`, `2m_to_10m`, `gte_10m`, `never`) | How often people message an agent that is still busy in their chat, and how long that message waits before the agent sees it. One row per human message that arrives while the chat's turn is running, recorded once its fate is known: `queued` (ran later as its own turn), `steered` (delivered into the running turn: `busy_input_mode: steer`, or an interrupt-mode correction that redirects the turn), `interrupted` (stopped the running turn, then ran as the next one), `merged` (folded into another busy message of the same burst; its wait is until that message reached the agent), `dropped` (never ran: cleared by `/stop` or `/new`, refused, or over the queue cap). The wait runs from arrival to its own turn starting or the steer landing, `never` when dropped. Commands, approval answers, internal events and background-process notifications are not counted. A message still queued when the gateway restarts is not counted: its saved row runs in the next process. The same holds for a message saved behind a paused queue (the user is told it is saved): it counts as `queued` only if this process later runs it, never as `dropped`. |Replies the relay connector carries report the platform the conversation liveson (the inbound's platform, else the platform the connector fronts when it@@ -650,7 +650,8 @@ database and the count is not reported.`gateway_peak_busy_chats_bucket` is the most distinct chats (session keys) of the profile that had aturn running at the same moment during that UTC day, as the gateway process serving the profile sawthem. The gateway keeps the running set and the day's maximum in memory per owning profile (the-profile whose scope claimed the turn, not the launch profile) and folds the maximum into the+profile whose scope claimed the turn, or the session's own profile for a chat auto-resumed at+startup; not the launch profile) and folds the maximum into theprofile's day state at each interaction; no chat id or session key is stored or sent. A profile nogateway served reads `0`; rows recorded before the field existed omit it.<!-- ---- end v5 engagement ---- -->