The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(telemetry): count busy messages that wait behind a running FIFO turn; cancelled rows read as dropped; per-profile home and closing-day peak
Tekniumbd47fe3b63a4
diff --git a/gateway/run_inbound.py b/gateway/run_inbound.pyindex 0875a2d4d77..58f521e4b5c 100644--- a/gateway/run_inbound.py+++ b/gateway/run_inbound.py@@ -37,7 +37,8 @@ from gateway.session import (neutralize_untrusted_inline_text,)from gateway.turn_lease import TurnLeaseTimeoutError-from hermes_cli.observability.shared_metrics_busy import note_busy_admitting, note_busy_outcome+from hermes_cli.observability.shared_metrics_busy import (+ note_busy_admitting, note_busy_outcome, settle_busy_arrival, stamp_busy_arrival)from typing import Any, Dict, List, Optional, Tupleif TYPE_CHECKING: # string annotations only; never imported at runtime (cycle)@@ -1217,28 +1218,17 @@ class GatewayInboundMixin(GatewayInboundHooksMixin, GatewayInboundTranscriptionMfrom gateway.session_authorities import active_authorityauthority = active_authority(self)if authority is not None and not event.get_command() and not is_internal:- from gateway.session_ingress import admit_message, executing_admission+ from gateway.session_ingress import executing_admissionif not executing_admission.get():- # The durable FIFO only orders turns; busy_input_mode still decides what a follow-up- # does to the RUNNING one (steer into it, redirect it, or end it so the FIFO advances).- # Without this, ``interrupt`` degrades to ``queue`` behind a turn that may hang for- # its whole request timeout.- # A bare "yes"/"no" to a blocking dangerous-command approval answers it (#46866):- # never an interrupt (read as deny) nor a queued turn (auto-deny deadlock).- # A turn a slash command produced (``/queue yes``) is never live typing.- if self._is_session_running(_quick_key) and not getattr(event, "_fifo_only", False) and (- await self._route_plaintext_approval_while_busy(event, _quick_key)- or await self._hm_busy_preempt(event, source, _quick_key)):- return True, None- _consumed, _consumer_reply = await self._hm_post_admission_consume(event, source, _quick_key)- if _consumed:- return True, _consumer_reply- if getattr(event, "_fifo_only", False) and event._stop_epoch != self._route_stop_epoch(_quick_key):- return True, None # a /stop or /new came after its command, as it clears the slot- # A failed admit stays the authority's answer (refusal notice or turn error), never a- # pending legacy turn: that would run unadmitted, outside the FIFO's order.- note_busy_admitting(event)- return True, await admit_message(authority, event)+ # A message the adapter saw as idle still waits when an earlier queued turn runs+ # outside its session guard: the running-turn check is the busy test that counts.+ busy = self._is_session_running(_quick_key) and not getattr(event, "_fifo_only", False)+ if busy:+ stamp_busy_arrival(self._delivery_adapter_for(source), event)+ try:+ return await self._hm_admit_to_fifo(authority, event, source, _quick_key, busy)+ finally:+ settle_busy_arrival(event) # never admitted, refused or dropped: its fate is knownif (authority is None and not is_internal and not event.get_command()and getattr(self, 'session_authority', None) is not None):# Scoped to a home this process does not serve: never fall back to the launch ledger.@@ -1246,6 +1236,31 @@ class GatewayInboundMixin(GatewayInboundHooksMixin, GatewayInboundTranscriptionMraise RuntimeStoreError('profile_mismatch')return False, None+ async def _hm_admit_to_fifo(+ self, authority: Any, event: MessageEvent, source: SessionSource, _quick_key: str, busy: bool,+ ) -> tuple[bool, Optional[str]]:+ """The authority-owned path of ``_hm_authority_admission`` (not inside a FIFO execution)."""+ from gateway.session_ingress import admit_message+ # The durable FIFO only orders turns; busy_input_mode still decides what a follow-up+ # does to the RUNNING one (steer into it, redirect it, or end it so the FIFO advances).+ # Without this, ``interrupt`` degrades to ``queue`` behind a turn that may hang for+ # its whole request timeout.+ # A bare "yes"/"no" to a blocking dangerous-command approval answers it (#46866):+ # never an interrupt (read as deny) nor a queued turn (auto-deny deadlock).+ # A turn a slash command produced (``/queue yes``) is never live typing.+ if busy and (await self._route_plaintext_approval_while_busy(event, _quick_key)+ or await self._hm_busy_preempt(event, source, _quick_key)):+ return True, None+ _consumed, _consumer_reply = await self._hm_post_admission_consume(event, source, _quick_key)+ if _consumed:+ return True, _consumer_reply+ if getattr(event, "_fifo_only", False) and event._stop_epoch != self._route_stop_epoch(_quick_key):+ return True, None # a /stop or /new came after its command, as it clears the slot+ # A failed admit stays the authority's answer (refusal notice or turn error), never a+ # pending legacy turn: that would run unadmitted, outside the FIFO's order.+ note_busy_admitting(event)+ return True, await admit_message(authority, event)+async def _handle_message(self, event: MessageEvent) -> Optional[str]:"""Handle an incoming message from any platform: auth → command check → running-agentinterrupt → get/create session → build context → run agent → return response."""diff --git a/gateway/session_authority.py b/gateway/session_authority.pyindex 1629fa5b5a4..adf903e7563 100644--- a/gateway/session_authority.py+++ b/gateway/session_authority.py@@ -21,6 +21,7 @@ from gateway.session_contract import (from gateway.session_events import SessionEventsfrom gateway.session_pending_controls import PendingControlsfrom gateway.session_producer_lifecycle import drop_producer_turn, mark_producer_admission+from hermes_cli.observability.shared_metrics_busy import busy_row_settledfrom hermes_state_runtime import (STEERING, RuntimeStoreError, admit_session_input, begin_runtime_epoch,cancel_session_input, claim_session_input, get_session_admission,@@ -833,6 +834,7 @@ class SessionAuthority:self.cancel_obligations.discard(admission_id)self.native_waiters.discard(admission_id)drop_producer_turn(self, admission_id)+ busy_row_settled(admission_id, outcome)waiter = self.waiters.pop(admission_id, None)if waiter is not None and not waiter.done():waiter.set_result(None)diff --git a/gateway/session_ingress.py b/gateway/session_ingress.pyindex 3f5691dc06f..32e04a36b8f 100644--- a/gateway/session_ingress.py+++ b/gateway/session_ingress.py@@ -77,7 +77,7 @@ async def admit_message(authority, event):from hermes_state_runtime import get_session_admissionrow = get_session_admission(authority.db, admission_id=receipt.admission_id)status = 'terminal' if row is None else row['status'] # None: deleted with its chat, nothing owed- note_busy_admitted(event, receipt.admission_id, status)+ note_busy_admitted(event, receipt.admission_id, status, None if row is None else row.get('outcome'))if status in ('terminal', 'unknown'):authority.native_waiters.discard(receipt.admission_id)if authority.waiters.get(receipt.admission_id) is waiter:diff --git a/hermes_cli/observability/shared_metrics_busy.py b/hermes_cli/observability/shared_metrics_busy.pyindex b34132b4006..93a16059bd1 100644--- a/hermes_cli/observability/shared_metrics_busy.py+++ b/hermes_cli/observability/shared_metrics_busy.py@@ -1,12 +1,15 @@"""Is the agent making people wait? Busy-chat arrivals and the day's peak of chats running at once.``hermes.gateway.busy_arrival.count``: one row per human message that arrives while its chat's turn-is still running (the adapter's busy path), recorded once its fate is known: it reached the agent-(its own turn started, or it was steered into the running one), it was folded into another busy-message, or it never ran. Commands, internal events, plain-text approval answers and plugin-consumed-input are not counted. The arrival rides the event object itself (``_busy_arrival``), so the merge,-admission and settle seams need no lookup; the durable FIFO's admission id links it to the turn that-later runs the restored copy.+is still running (the adapter's busy path, or the runner's running-turn check at FIFO admission for a+message the adapter saw as idle because an earlier queued turn now runs outside its session guard),+recorded once its fate is known: it reached the agent (its own turn started, or it was steered into+the running one), it was folded into another busy message, or it never ran (cancelled by /stop or+/new, refused before its turn, deleted with its chat). Commands, internal events, plain-text approval+answers and plugin-consumed input are not counted. The arrival rides the event object itself+(``_busy_arrival``), so the merge, admission and settle seams need no lookup; the durable FIFO's+admission id links it to the turn that later runs the restored copy. It belongs to the profile that+runs the chat's turns (the event's routing identity), never one inferred from the chat id.``gateway_peak_busy_chats_bucket`` (on ``hermes.engagement.day.count``): the most distinct chats(session keys) that had a turn running at the same moment during the UTC day, per owning profile.@@ -33,6 +36,9 @@ BUSY_ARRIVAL_METRIC = "hermes.gateway.busy_arrival.count"BUSY_ARRIVAL_DISPOSITIONS = frozenset({"queued", "steered", "interrupted", "merged", "dropped", "other"})BUSY_WAIT_BUCKETS = frozenset({"lt_5s", "5s_to_30s", "30s_to_2m", "2m_to_10m", "gte_10m", "never"})BUSY_PEAK_FIELD = "gateway_peak_busy_chats_bucket"+# Terminal FIFO outcomes of a row that reached the agent; any other (``cancelled``, ``rejected``, a+# deleted row) never ran.+_REACHED_OUTCOMES = frozenset({"completed", "interrupted", "failed", "steered", "steering"})_WAIT_THRESHOLDS = ((5.0, "lt_5s"), (30.0, "5s_to_30s"), (120.0, "30s_to_2m"), (600.0, "2m_to_10m"))@@ -80,15 +86,20 @@ def _tag(event: Any) -> _Arrival | None:def stamp_busy_arrival(adapter: Any, event: Any) -> None:- """A message reached the adapter while its chat is busy (``_handle_message_while_active`` hands- it to the shared runtime's busy dispatch, re-entered for the same event when a burst ends)."""+ """A message reached its chat while the chat is busy: the adapter's busy dispatch+ (``_handle_message_while_active``, re-entered for the same event when a burst ends), or the+ runner admitting it to the FIFO while the chat's turn runs. Stamped once. Its home is the+ routing identity's runtime profile (pinned at ingress), whose collection setting decides."""if getattr(event, "_busy_arrival", None) is not None or getattr(event, "internal", False) or event.get_command():returntry:- from .shared_metrics_gateway import _chat_home, _source_platform+ from gateway.session_identity import identity_of- chat_id = getattr(event.source, "chat_id", None)- event._busy_arrival = _Arrival(_source_platform(adapter, chat_id), _chat_home(adapter, chat_id))+ from .shared_metrics_gateway import _source_platform++ identity = identity_of(event.source)+ home = str(identity.runtime_home) if identity is not None else None+ event._busy_arrival = _Arrival(_source_platform(adapter, getattr(event.source, "chat_id", None)), home)except Exception:logger.debug("Busy arrival not stamped", exc_info=True)@@ -128,8 +139,10 @@ def _bind_home(tag: _Arrival) -> None:tag.home = str(get_hermes_home())-def note_busy_admitted(event: Any, admission_id: str, status: str) -> None:- """The row is committed; ``status`` as read right after. A drain that already claimed it ran it."""+def note_busy_admitted(event: Any, admission_id: str, status: str, outcome: Any = None) -> None:+ """The row is committed; ``status`` (and a terminal row's ``outcome``) as read right after. A+ drain that already claimed it ran it; a row already cancelled (/stop, /new, a pre-claim refusal)+ or deleted with its chat (``outcome`` None) never did."""tag = _tag(event)if tag is None:return@@ -138,8 +151,22 @@ def note_busy_admitted(event: Any, admission_id: str, status: str) -> None:_queued[tag.admission_id] = tagwhile len(_queued) > _QUEUED_MAX:_queued.popitem(last=False)[1].done = True- elif status in ("started", "unknown", "terminal"): # ``unknown``: it started, then its turn was lost- _resolve(tag, tag.disposition, time.monotonic())+ elif status in ("started", "unknown") or (status == "terminal" and outcome in _REACHED_OUTCOMES):+ _resolve(tag, tag.disposition, time.monotonic()) # ``unknown``: it started, then its turn was lost+ elif status == "terminal":+ _resolve(tag, "dropped", None)+++def busy_row_settled(admission_id: Any, outcome: str) -> None:+ """A queued row left the FIFO without a turn of its own (``SessionAuthority._settle_cancelled``):+ cancelled (/stop, /new, a refused head, an edit) never ran; a steered transfer reached the agent."""+ tag = _queued.pop(str(admission_id), None)+ if tag is None or tag.done:+ return+ if outcome == "steered":+ _resolve(tag, "steered", time.monotonic())+ else:+ _resolve(tag, "dropped", None)def note_busy_parked(event: Any) -> None:@@ -204,7 +231,8 @@ def _emit_busy_arrival(platform: Any, disposition: str, seconds: float | None) -_peak_lock = threading.Lock()_running: dict[str, str] = {} # session key -> owning profile home (resolved)-_peaks: dict[str, tuple[str, int]] = {} # home -> (UTC day, most chats running at once that day)+_peaks: dict[str, dict[str, int]] = {} # home -> UTC day -> most chats running at once that day+_PEAK_DAYS = 8 # a closing day's peak is kept until the profile's next interaction saves it_resolved: dict[str, str] = {}_now: Callable[[], float] = time.time # test / live-probe seam@@ -254,20 +282,21 @@ def observe_running_chats(session_keys: Any) -> None:counts[owner] = counts.get(owner, 0) + 1today = _day(_now())for owner, count in counts.items():- day, peak = _peaks.get(owner, (today, 0))- _peaks[owner] = (today, count if day != today else max(peak, count))+ days = _peaks.setdefault(owner, {})+ days[today] = max(days.get(today, 0), count)+ for stale in sorted(days)[:-_PEAK_DAYS]:+ del days[stale]except Exception:logger.debug("Running-chat peak not updated", exc_info=True)def day_peak(home: Any, day: str) -> int:"""Most chats of ``home``'s profile running at once on UTC ``day``, as seen by this process.- A later stored day means ``day`` closed with what was folded before; chats still running when- the day turned count toward the new one."""+ An earlier day keeps its own peak (the day closing after midnight reads it); chats still running+ when the day turned also count toward the new one."""owner = _home_key(home)with _peak_lock:running = sum(1 for h in _running.values() if h == owner)- stored_day, peak = _peaks.get(owner, (day, 0))- if stored_day == day:- return max(peak, running)- return running if stored_day < day else 0+ days = dict(_peaks.get(owner, {}))+ peak = days.get(day, 0)+ return max(peak, running) if not days or day >= max(days) else peakdiff --git a/hermes_cli/observability/shared_metrics_engagement.py b/hermes_cli/observability/shared_metrics_engagement.pyindex 7d8ff223dbb..e1455b5c72b 100644--- a/hermes_cli/observability/shared_metrics_engagement.py+++ b/hermes_cli/observability/shared_metrics_engagement.py@@ -201,6 +201,8 @@ def apply(if state is not None and not _valid_state(state):state = Noneif state is not None and state["day"] < today:+ if busy_peak is not None:+ _fold_busy_peak(state, busy_peak(state["day"])) # a peak reached just before midnight# The only close path. A stored day ahead of the clock (a step back across midnight) keeps# the interaction: the stored day never moves backwards, so no day is re-opened or emitted twice.rows, state = close_day(state), None@@ -219,11 +221,15 @@ def apply(key = " ".join(route)state["models"][key] = int(state["models"].get(key, 0)) + 1if busy_peak is not None:- stored = state.get(_BUSY_PEAK)- state[_BUSY_PEAK] = max(stored if isinstance(stored, int) else 0, busy_peak(today))+ _fold_busy_peak(state, busy_peak(today))return state, rows+def _fold_busy_peak(state: dict[str, Any], peak: int) -> None:+ stored = state.get(_BUSY_PEAK)+ state[_BUSY_PEAK] = max(stored if isinstance(stored, int) else 0, peak)++def _primary(models: dict[str, Any]) -> tuple[str, str]:counted = [(int(n), key) for key, n in models.items() if isinstance(n, int) and key.count(" ") == 1]if not counted:diff --git a/tests/hermes_cli/test_shared_metrics_busy.py b/tests/hermes_cli/test_shared_metrics_busy.pyindex f6648af6dd9..1cb66c95b65 100644--- a/tests/hermes_cli/test_shared_metrics_busy.py+++ b/tests/hermes_cli/test_shared_metrics_busy.py@@ -208,6 +208,7 @@ def test_a_busy_message_saved_behind_a_pause_is_not_dropped(rows, monkeypatch, r"""The user is told the message is saved and its FIFO row stays ``queued``: no ``dropped``row. When this process later runs that row (a ``session_busy`` pause lifting), it counts as``queued`` with its wait; after a restart the next process runs it and nothing is counted."""+ import gateway.session_authority # binds the real reader before the patch belowimport hermes_state_runtimefrom gateway.session_ingress import admit_messagefrom hermes_state_runtime import RuntimeStoreError@@ -265,3 +266,164 @@ def test_an_auto_resumed_chat_counts_toward_its_owning_profiles_peak(tmp_path, masyncio.run(resume())day = busy._day(DAY1)assert (busy.day_peak(beta, day), busy.day_peak(launch, day)) == (1, 0)+++def test_a_held_burst_dropped_by_stop_counts_under_its_own_profile_not_the_chats_last_one(rows, monkeypatch, tmp_path):+ """Two profiles serve one Discord channel through different bots; the other profile ran the+ chat's latest turn. A burst held for this profile's bot and dropped by /stop is recorded under+ the profile its routing identity names, so that profile's collection setting decides."""+ from gateway.session_identity import RoutingIdentity+ from hermes_constants import get_hermes_home, reset_hermes_home_override, set_hermes_home_override++ homes = []+ monkeypatch.setattr(rsm, "record_process_mark", lambda mark, data: homes.append(str(get_hermes_home())))+ alpha, beta = tmp_path / "alpha", tmp_path / "beta"+ token = set_hermes_home_override(str(beta))+ try:+ smg.start_reply_clock(SessionSource(platform=Platform.DISCORD, chat_id="chan", user_id="9"))+ finally:+ reset_hermes_home_override(token)+ source = SessionSource(platform=Platform.DISCORD, chat_id="chan", user_id="7")+ source._identity = RoutingIdentity(transport_profile="alpha", runtime_profile="alpha",+ authorization_home=alpha, runtime_home=alpha)++ async def scenario():+ adapter = _Adapter(handler=None)+ adapter.platform = Platform.DISCORD+ held = asyncio.create_task(dispatch_busy_burst(adapter, MessageEvent(text="x", source=source), KEY))+ await asyncio.sleep(0)+ discard_busy_burst(adapter, KEY)+ await held++ asyncio.run(scenario())+ smg.drain()+ assert homes == [str(alpha)]+++def test_a_closing_day_keeps_a_peak_reached_just_before_midnight(tmp_path, monkeypatch):+ """A third chat starts at 23:59:30; its first engagement event lands after midnight, and a turn+ boundary after midnight already started the new day. The closed day still reports 3."""+ from hermes_constants import reset_hermes_home_override, set_hermes_home_override++ monkeypatch.setattr(busy, "_running", {})+ monkeypatch.setattr(busy, "_peaks", {})+ late = datetime(2026, 9, 27, 23, 59, 0, tzinfo=UTC).timestamp()+ clock = {"now": late}+ monkeypatch.setattr(busy, "_now", lambda: clock["now"])+ monkeypatch.setattr(engagement, "_now", lambda: clock["now"])+ home = tmp_path / "a"++ def boundary(keys):+ token = set_hermes_home_override(str(home))+ try:+ busy.observe_running_chats(keys)+ finally:+ reset_hermes_home_override(token)++ boundary(["c1", "c2"])+ engagement.record(_store(home), RESOURCE, surface="gateway", route=None)+ clock["now"] = late + 30+ boundary(["c1", "c2", "c3"])+ clock["now"] = late + 90+ boundary(["c3"])+ engagement.record(_store(home), RESOURCE, surface="gateway", route=None)+ [row] = [c["dimensions"] for c in _store(home).counter_snapshot()+ if c["metric_name"] == contract.ENGAGEMENT_DAY_METRIC]+ assert row[contract.BUSY_PEAK_FIELD] == "3_to_5"+++async def _real_runner(monkeypatch):+ """Real GatewayRunner busy dispatch, real DiscordAdapter, real authority over a real SessionDB;+ the drain is not scheduled, so a queued row waits until the test runs or cancels it."""+ from gateway.config import GatewayConfig, PlatformConfig+ from gateway.platforms.base import SendResult+ from gateway.run import GatewayRunner+ from gateway.session_authority import SessionAuthority, initialize_session_authority+ from plugins.platforms.discord.adapter import DiscordAdapter++ monkeypatch.setattr(SessionAuthority, "_schedule", lambda self, ref: None)+ monkeypatch.setenv("DISCORD_ALLOWED_USERS", "42")+ runner = GatewayRunner(GatewayConfig())+ adapter = DiscordAdapter(PlatformConfig(enabled=True, token="fixture-token", typing_indicator=False))+ runner.adapters = {Platform.DISCORD: adapter}+ sent = []++ async def send(chat_id, content, **_kwargs):+ sent.append(content)+ return SendResult(success=True, message_id=str(len(sent)))+ monkeypatch.setattr(adapter, "send", send) # no network: notices and acks land here+ authority = await initialize_session_authority(runner, profile_id="default", instance_id="owner")+ runner._wire_adapter_handlers(adapter)+ source = adapter.build_source(chat_id="42", chat_type="dm", user_id="42")+ key = runner.session_store._generate_session_key(source)++ class RunningAgent:+ def interrupt(self, message=None):+ pass++ runner._session_state(key).turn.agent = RunningAgent() # an earlier queued turn is running+ return runner, authority, source, key+++def _busy_rows(rows):+ return [(d["disposition"], d["wait_bucket"]) for metric, d in rows() if metric == busy.BUSY_ARRIVAL_METRIC]+++async def test_a_message_the_adapter_saw_idle_counts_when_it_waits_behind_a_running_fifo_turn(rows, monkeypatch):+ """A's adapter task ended while queued B runs as a FIFO turn (outside the adapter's session+ guard): C reaches the runner as a fresh message, is admitted behind B, and counts once when+ its own turn starts."""+ from gateway.session_contract import SessionRef+ from gateway.session_authority import SessionAuthority+ from gateway.session_ingress_context import native_callback+ from hermes_constants import get_hermes_home++ runner, authority, source, key = await _real_runner(monkeypatch)+ monkeypatch.setattr(runner, "_effective_busy_input_mode", lambda source: "queue")+ late = MessageEvent(text="C", source=source, message_id="C")+ with native_callback(runner, late, get_hermes_home()):+ waiting = asyncio.create_task(runner._handle_message(late))+ for _ in range(200):+ if authority.waiters:+ break+ await asyncio.sleep(0.01)+ assert authority.waiters and _busy_rows(rows) == [] # still waiting behind B: not yet known+ runner._session_state(key).turn.agent = None # B finished; the drain runs C++ async def handle(event):+ return "ran"+ monkeypatch.setattr(runner, "_handle_message", handle)+ owner = runner.session_store.peek_session_id(key)+ await SessionAuthority._drain(authority, SessionRef(authority.profile_id, owner))+ waiting.cancel()+ await asyncio.gather(waiting, return_exceptions=True)+ assert _busy_rows(rows) == [("queued", "lt_5s")]+++async def test_a_busy_message_cancelled_before_admission_reads_its_row_counts_as_dropped(rows, monkeypatch):+ """/stop cancels the queued row while ``admit_message`` is still committing it: the re-read+ finds it terminal ``cancelled``. It never reached the agent: ``dropped`` / ``never``."""+ from gateway.session_ingress import admit_message+ from gateway.session_ingress_context import native_callback+ from hermes_constants import get_hermes_home++ runner, authority, source, _key = await _real_runner(monkeypatch)+ follow_up = MessageEvent(text="B", source=source, message_id="B")+ busy.stamp_busy_arrival(runner.adapters[Platform.DISCORD], follow_up)+ busy.note_busy_admitting(follow_up)+ commit = authority.admit_native++ async def commit_then_stop(event):+ receipt = await commit(event)+ stop = MessageEvent(text="/stop", source=source)+ with native_callback(runner, stop, get_hermes_home()):+ await runner._handle_message(stop)+ return receipt++ monkeypatch.setattr(authority, "admit_native", commit_then_stop)+ with native_callback(runner, follow_up, get_hermes_home()):+ assert await admit_message(authority, follow_up) is None+ busy.settle_busy_arrival(follow_up)+ assert _busy_rows(rows) == [("dropped", "never")]diff --git a/website/docs/developer-guide/relay-shared-metrics.md b/website/docs/developer-guide/relay-shared-metrics.mdindex 0a1d02a1791..cc26317befe 100644--- a/website/docs/developer-guide/relay-shared-metrics.md+++ b/website/docs/developer-guide/relay-shared-metrics.md@@ -496,7 +496,7 @@ itself ships.| `hermes.update.run` | kind, outcome, failed_stage, duration_bucket, from_version_age_bucket, apply_mode | Whether updates succeed, how long they take, where they fail, and how stale the version being updated from was. `outcome` is `success`, `noop`, `failed`, `refused`, or `partial` (committed, then interrupted or left the user's stashed changes parked; `hermes update` rows only). A `partial` run counts as an attempt and as a failure in the per-user update rate (these runs read `failed` before, so the trend stays continuous), and its reasons sit with the runs that applied the code and then failed a follow-up. `hermes update` rows are derived from the final update receipt, once per run (`kind` is `desktop` when Desktop's source-checkout hand-off ran it); a run the pre-update interpreter finishes, or the bare bootstrap interpreter finishes after dependency preparation failed (it cannot load the config reader), is parked locally with only the bounded fields the row needs and counted by the next start, which purges it unrecorded when collection is off (nothing is parked when that interpreter can already tell collection is off); Desktop packaged self-updates (`apply_mode=package`) are reported once by the app, after the restart that applies them. A lock refusal (`lock_held`) of the update child a Desktop hand-off launched (the update marker names it, or its parent, as the hand-off's delegate) reads `failed`, not `refused`: the Desktop already quit to run that update and reports it as a failed one. That includes the checkout lock held by another mutator (a Desktop build, a venv sync), not only another update running. || `hermes.update.stage` | stage, outcome, duration_bucket | Per-stage result and wall time of `hermes update` (plan, snapshot, apply, deps, build, restart, verify), from the receipt's stage timestamps. || `hermes.process.exit` | process_kind, exit_kind, crash_class | How CLI / TUI / gateway / serve / cron-tick processes end (`clean`, `crash` with an exception family only, `killed`, `watchdog`), reported by the next start in the same profile from a local marker (a start with collection off deletes these markers, and pending `provider_setup` ones, unreported). Turns aborted by a turn watchdog also count as `exit_kind=watchdog`. |-| `hermes.gateway.busy_arrival.count` | platform, disposition (`queued`/`steered`/`interrupted`/`merged`/`dropped`/`other`), wait bucket (`lt_5s`, `5s_to_30s`, `30s_to_2m`, `2m_to_10m`, `gte_10m`, `never`) | How often people message an agent that is still busy in their chat, and how long that message waits before the agent sees it. One row per human message that arrives while the chat's turn is running, recorded once its fate is known: `queued` (ran later as its own turn), `steered` (delivered into the running turn: `busy_input_mode: steer`, or an interrupt-mode correction that redirects the turn), `interrupted` (stopped the running turn, then ran as the next one), `merged` (folded into another busy message of the same burst; its wait is until that message reached the agent), `dropped` (never ran: cleared by `/stop` or `/new`, refused, or over the queue cap). The wait runs from arrival to its own turn starting or the steer landing, `never` when dropped. Commands, approval answers, internal events and background-process notifications are not counted. A message still queued when the gateway restarts is not counted: its saved row runs in the next process. The same holds for a message saved behind a paused queue (the user is told it is saved): it counts as `queued` only if this process later runs it, never as `dropped`. |+| `hermes.gateway.busy_arrival.count` | platform, disposition (`queued`/`steered`/`interrupted`/`merged`/`dropped`/`other`), wait bucket (`lt_5s`, `5s_to_30s`, `30s_to_2m`, `2m_to_10m`, `gte_10m`, `never`) | How often people message an agent that is still busy in their chat, and how long that message waits before the agent sees it. One row per human message that arrives while the chat's turn is running (including one that arrives while an earlier queued message's turn runs), counted under the profile that runs the chat (that profile's collection setting decides), recorded once its fate is known: `queued` (ran later as its own turn), `steered` (delivered into the running turn: `busy_input_mode: steer`, or an interrupt-mode correction that redirects the turn), `interrupted` (stopped the running turn, then ran as the next one), `merged` (folded into another busy message of the same burst; its wait is until that message reached the agent), `dropped` (never ran: cleared by `/stop` or `/new`, refused, deleted with its chat, or over the queue cap). The wait runs from arrival to its own turn starting or the steer landing, `never` when dropped. Commands, approval answers, internal events and background-process notifications are not counted. A message still queued when the gateway restarts is not counted: its saved row runs in the next process. The same holds for a message saved behind a paused queue (the user is told it is saved): it counts as `queued` only if this process later runs it, never as `dropped`. |Replies the relay connector carries report the platform the conversation liveson (the inbound's platform, else the platform the connector fronts when it@@ -652,7 +652,7 @@ turn running at the same moment during that UTC day, as the gateway process servthem. The gateway keeps the running set and the day's maximum in memory per owning profile (theprofile whose scope claimed the turn, or the session's own profile for a chat auto-resumed atstartup; not the launch profile) and folds the maximum into the-profile's day state at each interaction; no chat id or session key is stored or sent. A profile no+profile's day state at each interaction; a day that closes after midnight still reads its own maximum; no chat id or session key is stored or sent. A profile nogateway served reads `0`; rows recorded before the field existed omit it.<!-- ---- end v5 engagement ---- -->