Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
Report per-message telemetry events and activity tiers (#719)
Christian McDermott212520c4d444
diff --git a/core/switch_core/clients/agent_consumer.py b/core/switch_core/clients/agent_consumer.pyindex 54dee2e16..662a951a0 100644--- a/core/switch_core/clients/agent_consumer.py+++ b/core/switch_core/clients/agent_consumer.py@@ -97,6 +97,9 @@ from switch_core.events import (TaskUpdate,)from switch_core.gateway.known_agents import known_agent_for+from switch_core.observability.catalogue import OBSERVER_FAILURES+from switch_core.observability.metrics import metrics+from switch_core.observability.throttle import WarningThrottlefrom switch_core.transport import (InboundMedia,InboundMembership,@@ -107,8 +110,14 @@ from switch_core.transport import (if TYPE_CHECKING:from sqlalchemy.ext.asyncio import AsyncSession+ from switch_core.telemetry.messages import MessageTelemetry+logger = logging.getLogger(__name__)+# One for the process rather than one per consumer: every agent has a+# consumer, and a telemetry bug breaks the report for all of them.+_report_failures = WarningThrottle(60.0)+# How long to hold an incomplete multi-attachment group before delivering the# parts that did arrive, flagged as incomplete. Groups normally complete in# milliseconds (one sender, back-to-back events); this is a safety net so a@@ -185,6 +194,11 @@ class HostedNote:refusal: str | Nonedeliver: bool+ @property+ def redelivered(self) -> bool:+ """The mailbox already held this event: it was seen here before."""+ return self.refusal is None and not self.deliver+_HOSTED_STOPPED_MESSAGE = ("My cloud worker is stopped, so I did not process this message. "@@ -390,6 +404,7 @@ class AgentConsumer(Consumer[AgentActor]):hosted_launch_store: HostedLaunchStore,connections: AgentConnectionRegistry,frontend_base_url: str | None,+ message_telemetry: MessageTelemetry,) -> None:super().__init__(actor=actor)self._event_buffer = event_buffer@@ -405,6 +420,7 @@ class AgentConsumer(Consumer[AgentActor]):self._waking_notice_revisions: dict[str, int] = {}self._unreachable_notice_revisions: dict[str, int] = {}self._connections = connections+ self._message_telemetry = message_telemetryself._frontend_base_url = (frontend_base_url.rstrip("/") if frontend_base_url else None)@@ -589,6 +605,7 @@ class AgentConsumer(Consumer[AgentActor]):# that took a slot per question would multiply a single message into# dozens of concurrent checkouts.is_addressed = False+ agent_live = Falserefusal: str | None = Noneunavailable: str | None = Noneif self._addressed_without_lookup(event, meta) is not False:@@ -599,14 +616,14 @@ class AgentConsumer(Consumer[AgentActor]):gate = await self._gate_addressed(session, agent, event, meta)is_addressed = gate.addressedrefusal = gate.refusal- if (- is_addressed- and not self._triggered_by_auto_reply(event)- and not await self._is_available(session, agent, meta.room_id)- ):- unavailable = await self._reply_when_unavailable_here(- session, agent, meta, self._sender_handle(event)+ if is_addressed and not self._triggered_by_auto_reply(event):+ agent_live = await self._is_available(+ session, agent, meta.room_id)+ if not agent_live:+ unavailable = await self._reply_when_unavailable_here(+ session, agent, meta, self._sender_handle(event)+ )text = event.body@@ -680,6 +697,10 @@ class AgentConsumer(Consumer[AgentActor]):):self._unreachable_notice_revisions[meta.room_id] = launch.revisionunavailable = NOTICE_MESSAGES["unreachable"]+ if is_addressed:+ self._report_addressed(+ meta, event, hosted, agent_live=agent_live, has_attachment=False+ )if refusal is not None:await self._post_auto_reply(room.room_id, event, refusal, reply_thread_root)@@ -883,10 +904,19 @@ class AgentConsumer(Consumer[AgentActor]):) -> None:reply_thread_root = thread_id if thread_id is not None else event.event_idgated: Agent | None = None+ agent_live = Falseif is_addressed:async with self.session_factory() as session:agent = await self._fresh_agent(session)gate = await self._gate_addressed(session, agent, event, meta)+ # Only telemetry reads this here — the media path posts no+ # unavailable notice — so it is not worth a query while off.+ if (+ gate.addressed+ and self._message_telemetry.enabled+ and not self._triggered_by_auto_reply(event)+ ):+ agent_live = await self._is_available(session, agent, meta.room_id)if gate.addressed:gated = agentis_addressed = gate.addressed@@ -894,7 +924,6 @@ class AgentConsumer(Consumer[AgentActor]):await self._post_auto_reply(room.room_id, event, gate.refusal, reply_thread_root)-sender_kind: str | None = Noneon_behalf_of: str | None = Noneif PLATFORM_MARKER in event.content:@@ -925,14 +954,19 @@ class AgentConsumer(Consumer[AgentActor]):),)+ hosted: HostedNote | None = Noneif gated is not None:hosted = await self._note_hosted_addressed(gated, agent_event)if hosted is not None and hosted.refusal is not None:await self._post_auto_reply(room.room_id, event, hosted.refusal, reply_thread_root)- if hosted is not None and not hosted.deliver:- return+ if is_addressed:+ self._report_addressed(+ meta, event, hosted, agent_live=agent_live, has_attachment=True+ )+ if hosted is not None and not hosted.deliver:+ returnif logger.isEnabledFor(logging.DEBUG):logger.debug(@@ -1705,6 +1739,62 @@ class AgentConsumer(Consumer[AgentActor]):def _triggered_by_auto_reply(event: InboundMessage) -> bool:return bool(event.content.get(AUTO_REPLY_FLAG))+ def _report_addressed(+ self,+ meta: RoomMeta,+ event: InboundMessage,+ hosted: HostedNote | None,+ *,+ agent_live: bool,+ has_attachment: bool,+ ) -> None:+ """Count a message this agent was asked to act on.++ Every message let through the addressing policy and budget, whether or+ not the agent could take it, so an offline or stopped agent is still+ asked. `agent_live` is the agent having a live session for the room,+ and false as well when a hosted agent's mailbox refused the message:+ a full mailbox or a lost provider turns it away even with a worker+ attached. Not an auto-reply: Switch's notice that another agent is+ offline or refused is not someone asking this one for something. Not+ a redelivery the mailbox already holds: that request was counted when+ it first arrived. Not the files of a multi-file post delivered without+ its first part, which happens when a group times out incomplete and+ the rest arrives later: the post is counted with its first part, as+ the transport counts it, so it is one request however it is split.++ Guarded like the transport's observer call, and counted the same way:+ this runs on the delivery path, ahead of the enqueue, and a telemetry+ bug must not cost the agent the message.+ """+ if self._triggered_by_auto_reply(event):+ return+ if hosted is not None and hosted.redelivered:+ return+ group = parse_attachment_group(event.content)+ if group is not None and group[1] != 0:+ return+ try:+ self._message_telemetry.agent_addressed(+ tenant_id=self.tenant_id,+ room_id=meta.room_id,+ sender_transport_user_id=event.sender,+ from_platform=PLATFORM_MARKER in event.content,+ agent_metadata=self.agent.metadata_,+ agent_live=agent_live and (hosted is None or hosted.refusal is None),+ has_attachment=has_attachment,+ )+ except Exception:+ metrics().increment(OBSERVER_FAILURES, {"actor": "agent"})+ if (failures := _report_failures.note()) is not None:+ logger.error(+ "Could not report a message addressed to agent %s; it is "+ "still delivered. %d failure(s) since the last error.",+ self.agent.name,+ failures,+ exc_info=True,+ )+async def _post_auto_reply(self,transport_room_id: str,diff --git a/core/switch_core/clients/client_factory.py b/core/switch_core/clients/client_factory.pyindex 8bc782f0f..05d13c035 100644--- a/core/switch_core/clients/client_factory.py+++ b/core/switch_core/clients/client_factory.py@@ -18,6 +18,7 @@ from switch_core.messages.notify import MessageListenerfrom switch_core.transport import MessageTransportfrom switch_core.transport.ephemeral import EphemeralBusfrom switch_core.transport.invites import InviteBus+from switch_core.transport.observer import ParticipantMessageObserverfrom switch_core.transport.postgres import PostgresTransportfrom switch_core.transport.room_cache import RoomDeliveryCache@@ -39,6 +40,7 @@ class ClientFactory:invites: InviteBus,ephemeral: EphemeralBus,room_cache: RoomDeliveryCache,+ message_observer: ParticipantMessageObserver,) -> None:self._client_store = client_storeself._session_factory = session_factory@@ -52,6 +54,7 @@ class ClientFactory:self._ephemeral = ephemeral# One for the process, so every member of a room shares it.self._room_cache = room_cache+ self._message_observer = message_observerself._registry: dict[str,tuple[@@ -127,4 +130,5 @@ class ClientFactory:invites=self._invites,ephemeral=self._ephemeral,room_cache=self._room_cache,+ message_observer=self._message_observer,)diff --git a/core/switch_core/main.py b/core/switch_core/main.pyindex 1745f1773..f9c1e731d 100644--- a/core/switch_core/main.py+++ b/core/switch_core/main.py@@ -6,7 +6,7 @@ import loggingimport osimport signalimport time-from collections.abc import AsyncIterator+from collections.abc import AsyncIterator, Coroutinefrom contextlib import asynccontextmanagerfrom functools import partialfrom pathlib import Path@@ -169,6 +169,7 @@ from switch_core.session_activity.maintenance import ()from switch_core.session_activity.outcomes import ApprovalOutcomesfrom switch_core.session_activity.service import AgentSessionActivityService+from switch_core.telemetry.messages import MessageTelemetryfrom switch_core.telemetry.reporter import SnapshotReporterfrom switch_core.telemetry.service import TelemetryServicefrom switch_core.telemetry.setup import build_telemetry@@ -288,29 +289,34 @@ async def _stop_management(management: Management) -> None:# The innermost of three nested budgets: under# `observability.logs.SHUTDOWN_FLUSH_SECONDS`, itself under-# `_FORCED_EXIT_GRACE_SECONDS`.+# `_FORCED_EXIT_GRACE_SECONDS`. This and the log flush already fill that+# window, so neither can grow. The per-message worker gets at most+# `_MESSAGE_TELEMETRY_DRAIN_SECONDS` and the sink the rest, so a slow worker+# can never cost the sink its final flush, which carries every kind of event,+# and whatever the worker does not use is the sink's._TELEMETRY_DRAIN_SECONDS = 1.0+_MESSAGE_TELEMETRY_DRAIN_SECONDS = 0.4async def _drain_telemetry(- telemetry: TelemetryService, http_client: httpx.AsyncClient | None+ telemetry: TelemetryService,+ message_telemetry: MessageTelemetry,+ http_client: httpx.AsyncClient | None,) -> None:"""Let in-flight product events finish, then close their client.Never raises and never overruns: a relay that stopped answering must nothold the process past the point where it is killed."""- try:- async with asyncio.timeout(_TELEMETRY_DRAIN_SECONDS):- await telemetry.aclose()- except TimeoutError:- logger.warning(- "Gave up waiting for in-flight telemetry after %.1fs; those events "- "are lost.",- _TELEMETRY_DRAIN_SECONDS,- )- except Exception:- logger.warning("Telemetry shutdown failed; continuing.", exc_info=True)+ loop = asyncio.get_running_loop()+ deadline = loop.time() + _TELEMETRY_DRAIN_SECONDS+ # First, because what it still holds is emitted through the sink.+ await _close_by(+ message_telemetry.aclose(),+ loop.time() + _MESSAGE_TELEMETRY_DRAIN_SECONDS,+ "queued message events",+ )+ await _close_by(telemetry.aclose(), deadline, "buffered telemetry events")if http_client is not None:try:await http_client.aclose()@@ -318,6 +324,22 @@ async def _drain_telemetry(logger.warning("Telemetry HTTP client did not close.", exc_info=True)+async def _close_by(+ closing: Coroutine[object, object, None], deadline: float, what: str+) -> None:+ """Await one telemetry shutdown step until `deadline`, on the loop's clock.+ Never raises. The step logs how much it lost itself."""+ try:+ async with asyncio.timeout_at(deadline):+ await closing+ except TimeoutError:+ logger.warning("Gave up waiting for %s at shutdown.", what)+ except Exception:+ logger.warning(+ "Reporting %s at shutdown failed; continuing.", what, exc_info=True+ )++async def _snapshot_loop(reporter: SnapshotReporter) -> None:# `no_tenant` for the reason every other long-lived task does it: the# snapshot binds each tenant in turn as it counts, and must not inherit@@ -519,6 +541,9 @@ async def run(config: SwitchConfig) -> None:telemetry, installed_at, telemetry_http = await build_telemetry(config, session_factory, switch_core_version())+ message_telemetry = MessageTelemetry(+ telemetry=telemetry, session_factory=session_factory+ )# ── Switch Trust guardrails ──────────────────────────────────────────────trust_client, trust_http = build_trust_client(config)@@ -594,6 +619,7 @@ async def run(config: SwitchConfig) -> None:invites=invites,ephemeral=ephemeral,room_cache=room_cache,+ message_observer=message_telemetry,)client_factory.register("agent",@@ -611,6 +637,7 @@ async def run(config: SwitchConfig) -> None:hosted_launch_store=HostedLaunchStore(),connections=connections,frontend_base_url=config.frontend_base_url,+ message_telemetry=message_telemetry,)# Members that only write: a person on another platform, and a bridge's own# identity (its reader, the WorkspaceConsumer, is built by the bridge).@@ -941,7 +968,7 @@ async def run(config: SwitchConfig) -> None:# teardown runs inside `_FORCED_EXIT_GRACE_SECONDS` and a# product event is the least valuable thing in it.await protocol.sessions.aclose()- await _drain_telemetry(telemetry, telemetry_http)+ await _drain_telemetry(telemetry, message_telemetry, telemetry_http)if trust_http is not None:await trust_http.aclose()await observability.aclose()diff --git a/core/switch_core/observability/catalogue.py b/core/switch_core/observability/catalogue.pyindex 6230b0ffe..85c19df77 100644--- a/core/switch_core/observability/catalogue.py+++ b/core/switch_core/observability/catalogue.py@@ -231,6 +231,17 @@ DELIVERY_FAILURES = _spec("loop it was.","actor",)+OBSERVER_FAILURES = _spec(+ "switch.messages.observer_failures",+ "sum",+ "{failure}",+ "Reports about a message that raised: the transport telling its observer "+ "after the commit, or an agent's consumer reporting a message addressed to "+ "it. The message is unaffected and only the report is lost, so these are "+ "caught by design and invisible without a counter. `actor` is who was "+ "writing or reading.",+ "actor",+)DELIVERY_LAG = _spec("switch.messages.delivery_lag","histogram",@@ -550,6 +561,7 @@ CATALOGUE: dict[str, MetricSpec] = {MESSAGES_DELIVERED,SEND_FAILURES,DELIVERY_FAILURES,+ OBSERVER_FAILURES,DELIVERY_LAG,DELIVERY_CACHE_READS,DELIVERY_CACHE_FILLS,diff --git a/core/switch_core/observability/otlp.py b/core/switch_core/observability/otlp.pyindex 4158487e8..831608d55 100644--- a/core/switch_core/observability/otlp.py+++ b/core/switch_core/observability/otlp.py@@ -100,6 +100,20 @@ class OtlpSendError(RuntimeError):"""A payload did not reach the collector."""+class OtlpPartialRejection(OtlpSendError):+ """The collector took the payload but refused `rejected` of its records.++ A subclass, so a caller that only cares whether everything arrived can+ keep catching :class:`OtlpSendError`; one that batches can tell how much+ of the batch was lost.+ """++ def __init__(self, url: str, rejected: int, reason: str) -> None:+ super().__init__(f"POST {url} rejected {rejected} record(s): {reason}")+ self.rejected = rejected+ self.reason = reason++def _otlp_value(value: AttributeValue) -> dict[str, Any]:# `bool` first: it subclasses `int`, and as 1/0 a yes/no becomes something# a receiver will average.@@ -414,7 +428,6 @@ def _raise_on_partial_rejection(url: str, response: httpx.Response) -> None:if not rejected or int(rejected) == 0:return- raise OtlpSendError(- f"POST {url} rejected {rejected} record(s): "- f"{partial.get('errorMessage') or 'no reason given'}"+ raise OtlpPartialRejection(+ url, int(rejected), partial.get("errorMessage") or "no reason given")diff --git a/core/switch_core/observability/throttle.py b/core/switch_core/observability/throttle.pynew file mode 100644index 000000000..995520f3e--- /dev/null+++ b/core/switch_core/observability/throttle.py@@ -0,0 +1,42 @@+"""A condition that can hold for every message, logged at most once a minute.++Code on the message path runs once per message, so a condition that persists+— a full buffer, a database that will not answer, an observer that raises —+would otherwise write one warning per message and bury the server's own log+at the moment an operator needs it most.+"""++from __future__ import annotations++import time+++class WarningThrottle:+ """Counts occurrences of one condition and says when to log them.++ `note` records an occurrence. It returns how many there have been since+ the last time it said to log, this one included, when the interval has+ passed, and None while it has not. The first occurrence always logs.++ `take_pending` hands back what was counted but not yet logged, for a+ caller about to stop that would otherwise lose the tally of a quiet+ stretch at the end.+ """++ def __init__(self, interval_seconds: float) -> None:+ self._interval = interval_seconds+ self._pending = 0+ self._last_logged: float | None = None++ def note(self) -> int | None:+ self._pending += 1+ now = time.monotonic()+ if self._last_logged is not None and now - self._last_logged < self._interval:+ return None+ count, self._pending = self._pending, 0+ self._last_logged = now+ return count++ def take_pending(self) -> int:+ count, self._pending = self._pending, 0+ return countdiff --git a/core/switch_core/telemetry/catalogue.py b/core/switch_core/telemetry/catalogue.pyindex 82a150434..900bff29f 100644--- a/core/switch_core/telemetry/catalogue.py+++ b/core/switch_core/telemetry/catalogue.py@@ -90,7 +90,9 @@ BRIDGE_PLATFORM = one_of("slack", "mattermost", "discord", "teams", "telegram", "none", "unknown")-CHANNEL_TYPE = one_of("channel_public", "channel_private", "direct", "none")+# `unknown` as for BRIDGE_PLATFORM: the room exists and the lookup that would+# have named its type failed, which is not the same as it having none.+CHANNEL_TYPE = one_of("channel_public", "channel_private", "direct", "none", "unknown")# All four connection models in `bridges/agent/protocol/types.py`. Missing one# drops the registration event for every agent that uses it.@@ -103,8 +105,18 @@ AGENT_TYPE = one_of(# agent registered without declaring one at all.KNOWN_AGENT_TYPE = one_of("claude-code", "codex", "opencode", "other", "none")+# The runtime as `agent_message_sent` reports it. That event looks the sending+# agent up after the fact, so it has a third answer: `unknown`, for a lookup+# that failed or found no agent.+LOOKED_UP_AGENT_TYPE = one_of(*KNOWN_AGENT_TYPE.values, "unknown")+ACTOR_KIND = one_of("user", "agent", "system")+# Who said a message. `platform` is Switch itself speaking — a template's+# kickoff, or a request it carries on a person's behalf — and `unknown` a+# sender whose client could not be looked up.+SENDER_KIND = one_of("user", "agent", "platform", "unknown")+OUTCOME = one_of("success", "failure")# The four built-in types. A user-defined type's slug is free text, so it@@ -196,6 +208,17 @@ _SNAPSHOT_COUNTS = ("turn_human_to_agent_1d","turn_agent_to_human_1d","turn_agent_to_agent_1d",+ # The tiers of activity, as chat identities. `chat_identity_active_*`+ # above is the most active (spoke in a room with an agent in it);+ # `chat_identity_posted_*` is anyone who said anything in any room;+ # `chat_identity_in_room_count` is everyone Switch has seen in a room: a+ # member of a live room, or anyone who posted in the last seven days. Not+ # the channels' full audience — Switch records a person only once they+ # post, are added by name, or join after the channel was adopted, so a+ # member who has only ever read is not in it.+ "chat_identity_posted_1d",+ "chat_identity_posted_7d",+ "chat_identity_in_room_count","attachment_count_1d",# Stock rather than flow. The `*_attached_count` figures sit beside the# totals because the gap between made and used is the signal.@@ -360,6 +383,56 @@ CATALOGUE: Mapping[str, Mapping[str, PropertyType]] = {# in the failure code and nothing alike in the time."duration_ms": NUMBER,},+ # ── Messages ─────────────────────────────────────────────────────────────+ # One per message, unlike everything above. They carry no identifier, so+ # they answer "how much, and between whom", never "who": distinct people+ # come from the snapshot's `user_*` and `chat_identity_*` counts. A multi-file post is one+ # message, not one per file.+ #+ # Everything a participant chose to say in a room — the population the+ # tenant is metered for. Notices Switch posts on someone's behalf are not+ # in it. The room sizes are its members right now, so averaging them over+ # these events gives the room size a typical message is said to — once+ # `bridge_platform = unknown` is filtered out, since a room that could not+ # be read reports its sizes as -1.+ "room_message_sent": {+ "sender_kind": SENDER_KIND,+ "bridge_platform": BRIDGE_PLATFORM,+ "channel_type": CHANNEL_TYPE,+ "room_user_count": NUMBER,+ "room_agent_count": NUMBER,+ "has_attachment": BOOLEAN,+ "in_thread": BOOLEAN,+ },+ # A message an agent was asked to act on: addressed to it and let through+ # its addressing policy and budget, whether or not the agent was there to+ # take it. One per agent addressed, so a message naming two agents is two+ # of these and one `room_message_sent`; a message a hosted agent's mailbox+ # already holds is not counted again, nor a multi-file post delivered in+ # two pieces. `agent_live` is whether the agent had a live session for the+ # room when it arrived: false for one that was offline, still starting, or+ # whose cloud worker refused the message, and always for a session_passive+ # agent, which reads its messages later rather than live.+ "agent_message_received": {+ "sender_kind": SENDER_KIND,+ "known_agent_type": KNOWN_AGENT_TYPE,+ "bridge_platform": BRIDGE_PLATFORM,+ "channel_type": CHANNEL_TYPE,+ "has_attachment": BOOLEAN,+ "agent_live": BOOLEAN,+ },+ # A message an agent posted: its replies, and anything else it chose to+ # say. Also a `room_message_sent` with `sender_kind = agent`; a separate+ # event so it can carry the runtime. `room_user_count` separates an agent+ # answering people from agents talking among themselves.+ "agent_message_sent": {+ "known_agent_type": LOOKED_UP_AGENT_TYPE,+ "bridge_platform": BRIDGE_PLATFORM,+ "channel_type": CHANNEL_TYPE,+ "room_user_count": NUMBER,+ "has_attachment": BOOLEAN,+ "in_thread": BOOLEAN,+ },# ── Resources, keys and groups ───────────────────────────────────────────# Creating is intent, attaching is use, and the gap between them is the# signal — hence both.diff --git a/core/switch_core/telemetry/messages.py b/core/switch_core/telemetry/messages.pynew file mode 100644index 000000000..a68497903--- /dev/null+++ b/core/switch_core/telemetry/messages.py@@ -0,0 +1,562 @@+"""The per-message events: what is said in rooms, and to and by agents.++Everything else in the catalogue fires a handful of times a day. These fire+once per message, so they are the only events where reporting could slow the+thing being reported on. Three rules keep it off that path:++- **The sender never waits.** The transport and the agent consumer hand over a+ small record and return. One worker task looks up what the event needs — the+ room's platform, type and membership, who the sender is — and emits.+- **Lookups are cached.** A busy room says a lot and changes membership+ rarely, so a room is read at most once per `_CACHE_TTL_SECONDS` rather than+ once per message. The member counts can therefore be that far behind. A+ lookup that finds nothing is cached too, as `unknown`: a room deleted while+ its last messages were queued must not cost a query per message.+- **A failing lookup is left alone.** A room, sender or agent whose lookup+ failed is not looked up again for `_LOOKUP_BACKOFF_SECONDS`, and events+ report it as `unknown` meanwhile. Only that key: one tenant or room that+ keeps failing must not blank every other. When lookups fail+ `_FAILURES_TO_PAUSE_ALL` times in a row, with none succeeding between, the+ database itself is in trouble and every lookup pauses for that long. Either+ way the failure is logged at most once a minute, rather than the worker+ retrying a struggling database, and logging a traceback, once per message.++The queue is bounded: a worker that falls behind drops events with a warning+rather than growing a backlog inside the process it is measuring.+"""++from __future__ import annotations++import asyncio+import contextvars+import logging+import time+from collections import OrderedDict+from dataclasses import dataclass++from sqlalchemy import and_, func, select+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker++from switch_core.db.models import Agent, Client, ClientRoom, CollaborationBridge, Room+from switch_core.db.session_scope import tenant_session+from switch_core.observability.throttle import WarningThrottle+from switch_core.telemetry.service import TelemetryService+from switch_core.telemetry.snapshot import (+ AGENT_CLIENT_TYPE,+ HUMAN_CLIENT_TYPE,+ normalise_channel_type,+ normalise_known_agent_type,+ normalise_platform,+)+from switch_core.transport.observer import ParticipantMessage++logger = logging.getLogger(__name__)++_QUEUE_SIZE = 10_000+_CACHE_TTL_SECONDS = 300.0+_CACHE_MAX_ENTRIES = 10_000+_WARNING_INTERVAL_SECONDS = 60.0+_LOOKUP_BACKOFF_SECONDS = 5.0+# Failures in a row, from any keys, that read as the database failing rather+# than one bad row or tenant. A success between them resets the count.+_FAILURES_TO_PAUSE_ALL = 3+# Inside `main._MESSAGE_TELEMETRY_DRAIN_SECONDS`, leaving the rest of it for+# cancelling the worker.+_DRAIN_SECONDS = 0.3++# The transport's `ActorRole` as the catalogue's `sender_kind`. A system or+# bridge writer is Switch itself.+_ROLE_KIND = {+ "human": "user",+ "agent": "agent",+ "system": "platform",+ "bridge": "platform",+}++# `clients.type` as `sender_kind`. `admin` is Switch's own client.+_CLIENT_TYPE_KIND = {+ HUMAN_CLIENT_TYPE: "user",+ AGENT_CLIENT_TYPE: "agent",+ "admin": "platform",+ "bridge": "platform",+}++# An agent whose runtime could not be looked up. Not `none`, which is a real+# answer: an agent that declares no runtime.+_UNKNOWN_RUNTIME = "unknown"+++@dataclass(frozen=True)+class _RoomFacts:+ bridge_platform: str+ channel_type: str+ user_count: int+ agent_count: int+++# A room whose lookup failed. Reported rather than dropped, so a lookup+# problem shows as `unknown` in the charts instead of as fewer messages; -1+# rather than 0 so it cannot be read as an empty room. The catalogue has no+# empty value, so an average of the counts has to filter on+# `bridge_platform != unknown` — the docs say so.+_UNKNOWN_ROOM = _RoomFacts(+ bridge_platform="unknown", channel_type="unknown", user_count=-1, agent_count=-1+)+++@dataclass(frozen=True)+class _AgentAddressed:+ tenant_id: str+ room_id: str+ sender_transport_user_id: str+ from_platform: bool+ known_agent_type: str+ agent_live: bool+ has_attachment: bool+++class _TtlCache[K, V]:+ """Least-recently-used, with entries expiring after a fixed age."""++ def __init__(self, ttl_seconds: float, max_entries: int) -> None:+ self._ttl = ttl_seconds+ self._max = max_entries+ self._entries: OrderedDict[K, tuple[float, V]] = OrderedDict()++ def get(self, key: K) -> V | None:+ entry = self._entries.get(key)+ if entry is None:+ return None+ expires_at, value = entry+ if expires_at < time.monotonic():+ del self._entries[key]+ return None+ self._entries.move_to_end(key)+ return value++ def put(self, key: K, value: V) -> None:+ self._entries[key] = (time.monotonic() + self._ttl, value)+ self._entries.move_to_end(key)+ while len(self._entries) > self._max:+ self._entries.popitem(last=False)+++class MessageTelemetry:+ """Reports `room_message_sent`, `agent_message_sent` and+ `agent_message_received`.++ The transport's `ParticipantMessageObserver`, and what the agent consumer+ tells when a message is addressed to its agent. Does nothing at all — no+ queue, no lookups — while telemetry is off.+ """++ def __init__(+ self,+ *,+ telemetry: TelemetryService,+ session_factory: async_sessionmaker[AsyncSession],+ ) -> None:+ self._telemetry = telemetry+ self._session_factory = session_factory+ self._queue: asyncio.Queue[ParticipantMessage | _AgentAddressed] = (+ asyncio.Queue(maxsize=_QUEUE_SIZE)+ )+ self._worker: asyncio.Task[None] | None = None+ self._closed = False+ self._rooms: _TtlCache[tuple[str, str], _RoomFacts] = _TtlCache(+ _CACHE_TTL_SECONDS, _CACHE_MAX_ENTRIES+ )+ self._sender_kinds: _TtlCache[tuple[str, str], str] = _TtlCache(+ _CACHE_TTL_SECONDS, _CACHE_MAX_ENTRIES+ )+ self._agent_runtimes: _TtlCache[tuple[str, str], str] = _TtlCache(+ _CACHE_TTL_SECONDS, _CACHE_MAX_ENTRIES+ )+ # Keys whose lookup failed, as `(lookup, tenant_id, id)`, until they+ # may be tried again.+ self._failed_lookups: _TtlCache[tuple[str, str, str], bool] = _TtlCache(+ _LOOKUP_BACKOFF_SECONDS, _CACHE_MAX_ENTRIES+ )+ self._failures_in_a_row = 0+ self._all_lookups_resume_at: float | None = None+ # Taken off the queue and not yet reported. Not `qsize`, which leaves+ # out the one the worker is cancelled in the middle of.+ self._in_hand = 0+ self._drops = WarningThrottle(_WARNING_INTERVAL_SECONDS)+ self._late = WarningThrottle(_WARNING_INTERVAL_SECONDS)+ self._report_failures = WarningThrottle(_WARNING_INTERVAL_SECONDS)+ self._lookup_failures = WarningThrottle(_WARNING_INTERVAL_SECONDS)+ self._lookup_pauses = WarningThrottle(_WARNING_INTERVAL_SECONDS)+ self._lookup_misses = WarningThrottle(_WARNING_INTERVAL_SECONDS)++ @property+ def enabled(self) -> bool:+ """Whether anything handed over is reported. A caller with work to do+ only for telemetry can skip it while this is false."""+ return self._telemetry.enabled++ # ── Called on the sender's path ──────────────────────────────────────────++ def observe(self, message: ParticipantMessage) -> None:+ """A participant said something in a room."""+ self._enqueue(message)++ def agent_addressed(+ self,+ *,+ tenant_id: str,+ room_id: str,+ sender_transport_user_id: str,+ from_platform: bool,+ agent_metadata: dict | None,+ agent_live: bool,+ has_attachment: bool,+ ) -> None:+ """A message was addressed to an agent and let through its addressing+ policy and budget. `agent_live` is whether the agent had a live session+ for the room when it arrived.++ The runtime is read from `agent_metadata` here rather than by the+ worker, so the queue holds a value and not the agent's own dict.+ """+ self._enqueue(+ _AgentAddressed(+ tenant_id=tenant_id,+ room_id=room_id,+ sender_transport_user_id=sender_transport_user_id,+ from_platform=from_platform,+ known_agent_type=normalise_known_agent_type(agent_metadata),+ agent_live=agent_live,+ has_attachment=has_attachment,+ )+ )++ def _enqueue(self, item: ParticipantMessage | _AgentAddressed) -> None:+ if not self._telemetry.enabled:+ return+ if self._closed:+ # Senders still run while shutdown drains, and what they say then+ # is lost; counted, so a low figure for the last minute has a cause.+ if (late := self._late.note()) is not None:+ logger.warning(+ "%d message event(s) arrived after message telemetry shut "+ "down and were dropped.",+ late,+ )+ return+ if self._worker is None or self._worker.done():+ # Before the put, so a worker that ended with the queue full is+ # replaced rather than leaving every later event to be dropped.+ # A fresh context: started from whichever sender enqueued first, it+ # would otherwise carry that sender's tenant and log fields for its+ # whole life.+ self._worker = asyncio.get_running_loop().create_task(+ self._run(), context=contextvars.Context()+ )+ try:+ self._queue.put_nowait(item)+ except asyncio.QueueFull:+ if (dropped := self._drops.note()) is not None:+ self._warn_dropped(dropped)++ @staticmethod+ def _warn_dropped(dropped: int) -> None:+ logger.warning(+ "Message telemetry is behind: %d event(s) dropped because %d were "+ "already queued. Message counts in analytics will read low.",+ dropped,+ _QUEUE_SIZE,+ )++ # ── The worker ───────────────────────────────────────────────────────────++ async def _run(self) -> None:+ while True:+ item = await self._queue.get()+ self._in_hand += 1+ try:+ await self._report(item)+ except Exception:+ # One bad event must not stop the rest being reported, and a+ # bug that breaks every event must not log once per message.+ if (failures := self._report_failures.note()) is not None:+ logger.error(+ "Could not report a message event; %d event(s) dropped "+ "this way since the last error.",+ failures,+ exc_info=True,+ )+ finally:+ self._queue.task_done()+ # Not in the `finally`: an event the shutdown cancels mid-report+ # was never reported, and leaving it counted is what says so.+ self._in_hand -= 1++ async def _report(self, item: ParticipantMessage | _AgentAddressed) -> None:+ if isinstance(item, _AgentAddressed):+ await self._report_addressed(item)+ else:+ await self._report_message(item)++ async def _report_message(self, message: ParticipantMessage) -> None:+ room = await self._room_facts(message.tenant_id, message.room_id)+ self._telemetry.emit(+ "room_message_sent",+ sender_kind=_ROLE_KIND.get(message.sender_role, "unknown"),+ bridge_platform=room.bridge_platform,+ channel_type=room.channel_type,+ room_user_count=room.user_count,+ room_agent_count=room.agent_count,+ has_attachment=message.has_attachment,+ in_thread=message.in_thread,+ )+ if message.sender_role != "agent":+ return+ self._telemetry.emit(+ "agent_message_sent",+ known_agent_type=await self._agent_runtime(+ message.tenant_id, message.sender_client_id+ ),+ bridge_platform=room.bridge_platform,+ channel_type=room.channel_type,+ room_user_count=room.user_count,+ has_attachment=message.has_attachment,+ in_thread=message.in_thread,+ )++ async def _report_addressed(self, item: _AgentAddressed) -> None:+ room = await self._room_facts(item.tenant_id, item.room_id)+ sender_kind = (+ "platform"+ if item.from_platform+ else await self._sender_kind(item.tenant_id, item.sender_transport_user_id)+ )+ self._telemetry.emit(+ "agent_message_received",+ sender_kind=sender_kind,+ known_agent_type=item.known_agent_type,+ bridge_platform=room.bridge_platform,+ channel_type=room.channel_type,+ has_attachment=item.has_attachment,+ agent_live=item.agent_live,+ )++ async def aclose(self) -> None:+ """Report what is already queued, briefly, then stop."""+ self._closed = True+ try:+ if self._worker is not None:+ try:+ async with asyncio.timeout(_DRAIN_SECONDS):+ await self._queue.join()+ except TimeoutError:+ pass+ finally:+ self._worker.cancel()+ # `wait` rather than awaiting the task: it does not raise the+ # worker's own cancellation, so nothing here has to suppress+ # one, and a cancellation aimed at this caller — the shutdown+ # timeout — still reaches it. Outside the `finally`, so a+ # cancellation that lands during the drain is not followed by+ # an unbounded wait.+ await asyncio.wait({self._worker})+ finally:+ self._report_unsent()++ def _report_unsent(self) -> None:+ """Say what shutdown could not report. Run from a `finally`, so the+ caller's timeout cutting `aclose` off still leaves the count."""+ if unreported := self._queue.qsize() + self._in_hand:+ logger.warning(+ "%d message event(s) still queued at shutdown were dropped.",+ unreported,+ )+ if dropped := self._drops.take_pending():+ self._warn_dropped(dropped)+ if late := self._late.take_pending():+ logger.warning(+ "%d more message event(s) arrived after message telemetry shut "+ "down and were dropped.",+ late,+ )+ if failures := self._report_failures.take_pending():+ logger.error(+ "%d more message event(s) could not be reported since the last "+ "error and were dropped.",+ failures,+ )++ # ── Lookups ──────────────────────────────────────────────────────────────++ def _may_look_up(self, key: tuple[str, str, str]) -> bool:+ """Whether `key` may be looked up now: neither it nor the database as a+ whole failed within the last `_LOOKUP_BACKOFF_SECONDS`."""+ if (+ self._all_lookups_resume_at is not None+ and time.monotonic() < self._all_lookups_resume_at+ ):+ return False+ return self._failed_lookups.get(key) is None++ def _lookup_answered(self) -> None:+ """The database answered, whether or not it found anything."""+ self._failures_in_a_row = 0++ def _lookup_failed(self, key: tuple[str, str, str], what: str) -> None:+ """Back off after a failed lookup, and say so at most once a minute.++ Called from inside the `except` block, so the warning carries the+ traceback of the failure that triggered it.+ """+ self._failed_lookups.put(key, True)+ self._failures_in_a_row += 1+ if (failures := self._lookup_failures.note()) is not None:+ logger.warning(+ "Message telemetry could not look up %s, so events report it as "+ "unknown and it is not looked up again for %.0fs; %d lookup(s) "+ "failed since the last warning.",+ what,+ _LOOKUP_BACKOFF_SECONDS,+ failures,+ exc_info=True,+ )+ # Not reset when it trips: while the database stays down, the first+ # lookup after each pause fails and starts the next one at once.+ if self._failures_in_a_row >= _FAILURES_TO_PAUSE_ALL:+ self._all_lookups_resume_at = time.monotonic() + _LOOKUP_BACKOFF_SECONDS+ if (pauses := self._lookup_pauses.note()) is not None:+ logger.warning(+ "Message telemetry lookups failed %d times in a row, so every "+ "lookup pauses for %.0fs and events report the rooms and "+ "senders they need as unknown; %d pause(s) since the last "+ "warning.",+ self._failures_in_a_row,+ _LOOKUP_BACKOFF_SECONDS,+ pauses,+ )++ def _lookup_missed(self, what: str) -> None:+ if (misses := self._lookup_misses.note()) is not None:+ logger.warning(+ "Message telemetry found no %s, so events report it as unknown; "+ "%d lookup(s) found nothing since the last warning.",+ what,+ misses,+ )++ async def _room_facts(self, tenant_id: str, room_id: str) -> _RoomFacts:+ key = (tenant_id, room_id)+ cached = self._rooms.get(key)+ if cached is not None:+ return cached+ failure_key = ("room", tenant_id, room_id)+ if not self._may_look_up(failure_key):+ return _UNKNOWN_ROOM+ try:+ facts = await self._load_room_facts(tenant_id, room_id)+ except Exception:+ self._lookup_failed(failure_key, f"room {room_id}")+ return _UNKNOWN_ROOM+ self._lookup_answered()+ if facts is None:+ self._lookup_missed(f"room {room_id}")+ self._rooms.put(key, _UNKNOWN_ROOM)+ return _UNKNOWN_ROOM+ self._rooms.put(key, facts)+ return facts++ async def _load_room_facts(self, tenant_id: str, room_id: str) -> _RoomFacts | None:+ async with tenant_session(self._session_factory, tenant_id) as session:+ row = (+ await session.execute(+ select(Room.channel_type, CollaborationBridge.type)+ .select_from(Room)+ .outerjoin(+ CollaborationBridge,+ and_(+ CollaborationBridge.id == Room.bridge_id,+ CollaborationBridge.tenant_id == tenant_id,+ ),+ )+ .where(Room.tenant_id == tenant_id, Room.id == room_id)+ )+ ).one_or_none()+ if row is None:+ return None+ members = await session.execute(+ select(Client.type, func.count())+ .select_from(ClientRoom)+ .join(Client, Client.id == ClientRoom.client_id)+ .where(+ ClientRoom.tenant_id == tenant_id,+ ClientRoom.room_id == room_id,+ Client.tenant_id == tenant_id,+ )+ .group_by(Client.type)+ )+ by_type = {client_type: int(count) for client_type, count in members.all()}+ channel_type, platform = row+ return _RoomFacts(+ bridge_platform=normalise_platform(platform),+ channel_type=normalise_channel_type(channel_type),+ user_count=by_type.get(HUMAN_CLIENT_TYPE, 0),+ agent_count=by_type.get(AGENT_CLIENT_TYPE, 0),+ )++ async def _sender_kind(self, tenant_id: str, transport_user_id: str) -> str:+ key = (tenant_id, transport_user_id)+ cached = self._sender_kinds.get(key)+ if cached is not None:+ return cached+ failure_key = ("sender", tenant_id, transport_user_id)+ if not self._may_look_up(failure_key):+ return "unknown"+ try:+ async with tenant_session(self._session_factory, tenant_id) as session:+ client_type = (+ await session.execute(+ select(Client.type).where(+ Client.tenant_id == tenant_id,+ Client.transport_user_id == transport_user_id,+ )+ )+ ).scalar_one_or_none()+ except Exception:+ self._lookup_failed(failure_key, "a message sender")+ return "unknown"+ self._lookup_answered()+ if client_type is None:+ self._lookup_missed("client for a message sender")+ self._sender_kinds.put(key, "unknown")+ return "unknown"+ kind = _CLIENT_TYPE_KIND.get(client_type, "unknown")+ self._sender_kinds.put(key, kind)+ return kind++ async def _agent_runtime(self, tenant_id: str, client_id: str) -> str:+ key = (tenant_id, client_id)+ cached = self._agent_runtimes.get(key)+ if cached is not None:+ return cached+ failure_key = ("agent", tenant_id, client_id)+ if not self._may_look_up(failure_key):+ return _UNKNOWN_RUNTIME+ try:+ async with tenant_session(self._session_factory, tenant_id) as session:+ result = await session.execute(+ select(Agent.metadata_).where(+ Agent.tenant_id == tenant_id, Agent.client_id == client_id+ )+ )+ row = result.one_or_none()+ except Exception:+ self._lookup_failed(failure_key, "an agent's runtime")+ return _UNKNOWN_RUNTIME+ self._lookup_answered()+ if row is None:+ self._lookup_missed(f"agent for client {client_id}")+ self._agent_runtimes.put(key, _UNKNOWN_RUNTIME)+ return _UNKNOWN_RUNTIME+ runtime = normalise_known_agent_type(row[0])+ self._agent_runtimes.put(key, runtime)+ return runtimediff --git a/core/switch_core/telemetry/service.py b/core/switch_core/telemetry/service.pyindex 2fcc125fc..b0d440d6a 100644--- a/core/switch_core/telemetry/service.py+++ b/core/switch_core/telemetry/service.py@@ -1,8 +1,8 @@"""The one call every site uses to report an event.`emit` is fire-and-forget and synchronous to call: creating a room must not-wait on a relay. It validates against the catalogue first, then hands the send-to a background task.+wait on a relay. It validates against the catalogue first, then hands the+record to the sink, which only buffers it.The asymmetry is deliberate. **A bad event raises** — it is a programming errorand silence would defeat the catalogue. **A failed send does not** — it is@@ -71,7 +71,6 @@ class TelemetryService:self._resource["service.version"] = versionif environment:self._resource["deployment.environment"] = environment- self._tasks: set[asyncio.Task[None]] = set()@propertydef enabled(self) -> bool:@@ -127,40 +126,27 @@ class TelemetryService:def _dispatch(self, record: TelemetryRecord) -> None:try:- loop = asyncio.get_running_loop()+ asyncio.get_running_loop()except RuntimeError:# A management command or a sync helper; not its job to have one.logger.debug("Telemetry event %s not sent: no running event loop.", record.name)return-- task = loop.create_task(self._send(record))- # Held for the lifetime of the send. An un-referenced task can be- # garbage-collected mid-flight, which drops the event silently and is- # exactly the kind of bug telemetry code is bad at revealing.- self._tasks.add(task)- task.add_done_callback(self._tasks.discard)-- async def _send(self, record: TelemetryRecord) -> None:try:- await self._sink.send(record)- except asyncio.CancelledError:- raise+ self._sink.send(record)except Exception:- # A bug in the sink, not a bad relay — but this is a bare task, so- # letting it out only surfaces at some later collection.+ # A bug in the sink, not a bad relay. `emit` raises only for a+ # malformed event, so this must not reach the caller either.logger.exception("Telemetry sink failed on event %s", record.name)async def aclose(self) -> None:- """Let in-flight sends finish, then close the sink.+ """Close the sink, which posts what it still holds.Shutdown is the one time waiting is right: the events worth losingleast are the ones emitted just before the process goes away, and the- sink's own timeout already bounds how long this can take.+ caller's timeout bounds how long this can take."""- if self._tasks:- await asyncio.gather(*tuple(self._tasks), return_exceptions=True)await self._sink.aclose()diff --git a/core/switch_core/telemetry/setup.py b/core/switch_core/telemetry/setup.pyindex 3b065c8f6..5c348dfba 100644--- a/core/switch_core/telemetry/setup.py+++ b/core/switch_core/telemetry/setup.py@@ -22,6 +22,14 @@ from switch_core.telemetry.sink import NullSink, OtlpRelaySink, TelemetrySinklogger = logging.getLogger(__name__)+# How the relay sink batches. Five seconds keeps an event's delay well inside+# what analytics cares about; two hundred records is a request of a few tens of+# kilobytes. The buffer holds roughly a minute of a very busy server, so a+# relay that is down for longer loses events rather than growing memory.+_FLUSH_INTERVAL_SECONDS = 5.0+_MAX_BATCH = 200+_MAX_BUFFERED = 10_000+async def build_telemetry(config: SwitchConfig,@@ -92,7 +100,10 @@ async def build_telemetry(timeout_seconds=config.telemetry_timeout_seconds,headers={},client=http_client,- )+ ),+ flush_interval_seconds=_FLUSH_INTERVAL_SECONDS,+ max_batch=_MAX_BATCH,+ max_buffered=_MAX_BUFFERED,)logger.info("Product telemetry is ON: usage counts and timings are reported to "diff --git a/core/switch_core/telemetry/sink.py b/core/switch_core/telemetry/sink.pyindex 932a6518f..b063e0a72 100644--- a/core/switch_core/telemetry/sink.py+++ b/core/switch_core/telemetry/sink.py@@ -9,19 +9,24 @@ lets a reporting problem reach the caller.from __future__ import annotations+import asyncio+import contextvarsimport logging-from collections.abc import Mapping+from collections import deque+from collections.abc import Mapping, Sequencefrom dataclasses import dataclassfrom typing import Any, Protocolfrom switch_core.observability.otlp import (LogRecord,OtlpClient,+ OtlpPartialRejection,OtlpResource,OtlpSendError,build_logs_payload,otlp_attributes,)+from switch_core.observability.throttle import WarningThrottlefrom switch_core.telemetry.catalogue import PropertyValuelogger = logging.getLogger(__name__)@@ -31,6 +36,14 @@ logger = logging.getLogger(__name__)# receivers as an error._SEVERITY_INFO = 9+_DROP_WARNING_INTERVAL_SECONDS = 60.0++# How many batches shutdown posts at once. One at a time, a slow request+# already in flight would hold every batch behind it until the budget ran out;+# a handful rather than the fifty a full buffer makes, because the relay+# rate-limits on the sender's address.+_SHUTDOWN_CONCURRENCY = 4+@dataclass(frozen=True)class TelemetryRecord:@@ -48,9 +61,11 @@ class TelemetryRecord:class TelemetrySink(Protocol):- """The seam. One method, and it never raises."""+ """The seam. `send` hands a record over and returns at once: it never+ raises and never waits on the network, so it is safe to call once per+ message. Called with an event loop running."""- async def send(self, record: TelemetryRecord) -> None: ...+ def send(self, record: TelemetryRecord) -> None: ...async def aclose(self) -> None: ...@@ -59,7 +74,7 @@ class NullSink:"""Drops everything. Off is a sink that discards, so no call site has toask whether telemetry is on."""- async def send(self, record: TelemetryRecord) -> None:+ def send(self, record: TelemetryRecord) -> None:return Noneasync def aclose(self) -> None:@@ -67,7 +82,7 @@ class NullSink:class OtlpRelaySink:- """Posts one OTLP log record per event to the relay.+ """Posts events to the relay as OTLP log records, a batch at a time.Log records, not metrics: the relay routes on them and the analyticsexporter reads them as events. A metric would be dropped without complaint.@@ -78,45 +93,223 @@ class OtlpRelaySink:and then discarded. That is the easiest way to believe this works when itdoes not.- No batching and no retry, matching the operational exporter.+ **Batched**, because the message events fire once per message: a busy+ server posting one request per event would be a load the relay was never+ sized for, and would trip the rate limit it keys on the sender's address.+ `send` only buffers; one task posts every `flush_interval_seconds`, or as+ soon as `max_batch` records are waiting, and `aclose` posts what is left.+ A flush posts what was buffered when it began and no more: what arrives+ while it is posting waits for the next interval or a full batch, or a+ steady trickle would become a request per event again.++ No retry, matching the operational exporter, and the buffer is bounded: a+ relay that stops answering costs the events it misses, never memory.++ **Shutdown** posts what is left several batches at once, beside any request+ already in flight rather than behind it, and stops at the first batch the+ relay refuses. Whatever the caller's timeout cuts off, buffered or+ abandoned mid-request, is counted and logged."""- def __init__(self, *, client: OtlpClient) -> None:+ def __init__(+ self,+ *,+ client: OtlpClient,+ flush_interval_seconds: float,+ max_batch: int,+ max_buffered: int,+ ) -> None:self._client = client-- async def send(self, record: TelemetryRecord) -> None:- payload = build_logs_payload(- [- LogRecord(- # Datadog renders this as the log message, and a blank one- # makes the event unreadable there.- body=record.name,- severity_text="INFO",- severity_number=_SEVERITY_INFO,- time_nanos=record.timestamp_ns,- attributes={"event.name": record.name, **record.properties},+ self._flush_interval = flush_interval_seconds+ self._max_batch = max_batch+ self._max_buffered = max_buffered+ self._buffer: list[TelemetryRecord] = []+ # Taken from the buffer, waiting for a post.+ self._batches: deque[list[TelemetryRecord]] = deque()+ # Records in posts that have not finished.+ self._posting = 0+ self._wake = asyncio.Event()+ self._flusher: asyncio.Task[None] | None = None+ self._closing = False+ self._refused_at_shutdown = False+ self._drops = WarningThrottle(_DROP_WARNING_INTERVAL_SECONDS)+ self._late = WarningThrottle(_DROP_WARNING_INTERVAL_SECONDS)++ def send(self, record: TelemetryRecord) -> None:+ if self._closing:+ # Shutdown has already taken the final batch; posting this one+ # would race the client being closed.+ if (late := self._late.note()) is not None:+ logger.warning(+ "%d telemetry event(s), the latest %s, arrived after "+ "shutdown began and were dropped.",+ late,+ record.name,)- ],- _resource_from(record),+ return+ if self._flusher is None or self._flusher.done():+ # Before the capacity check, so a flusher that ended with the+ # buffer full is replaced rather than leaving every later event to+ # be dropped. A fresh context, so its warnings do not carry the log+ # fields of whichever request happened to emit first.+ self._flusher = asyncio.get_running_loop().create_task(+ self._run(), context=contextvars.Context()+ )+ if len(self._buffer) >= self._max_buffered:+ if (dropped := self._drops.note()) is not None:+ self._warn_dropped(dropped)+ return+ self._buffer.append(record)+ if len(self._buffer) >= self._max_batch:+ self._wake.set()++ def _warn_dropped(self, dropped: int) -> None:+ logger.warning(+ "Telemetry buffer is full (%d events waiting on the relay): %d "+ "event(s) dropped since the last warning.",+ self._max_buffered,+ dropped,)- _add_event_name(payload, record.name)- _add_relay_context(payload, record.resource)- try:- await self._client.post("logs", payload)- except OtlpSendError as exc:- # Switch has to keep working while analytics is down.- logger.warning(- "Telemetry event %s was not sent: %s. The event is dropped; "- "there is no retry.",- record.name,- exc,+ async def _run(self) -> None:+ while True:+ try:+ async with asyncio.timeout(self._flush_interval):+ await self._wake.wait()+ except TimeoutError:+ pass+ self._wake.clear()+ if self._closing:+ # `aclose` has taken the buffer and posts it.+ return+ self._take_buffer()+ await self._post_waiting()++ def _take_buffer(self) -> None:+ pending, self._buffer = self._buffer, []+ for start in range(0, len(pending), self._max_batch):+ self._batches.append(pending[start : start + self._max_batch])++ async def _post_waiting(self) -> None:+ """Post batches until none are waiting.++ Outside shutdown only the flusher takes from the buffer, and only as a+ flush begins, so this posts what that flush took and no more.+ """+ while self._batches and not self._refused_at_shutdown:+ batch = self._batches.popleft()+ self._posting += len(batch)+ try:+ await self._post(batch)+ except Exception:+ # A relay failure is already handled in `_post`; anything+ # reaching here is a bug in building the batch. Caught per+ # batch, so it costs neither the batches after it nor the task+ # every later event depends on.+ logger.exception(+ "Telemetry batch of %d event(s) could not be built; it is dropped.",+ len(batch),+ )+ # Not in a `finally`: a post the shutdown timeout cancels did not+ # send its records, and leaving them counted here is what reports+ # them as unsent.+ self._posting -= len(batch)++ async def _post(self, batch: Sequence[TelemetryRecord]) -> None:+ # Every record a service emits carries the same resource, so this is+ # one payload in practice; grouping keeps two services sharing a sink+ # from being reported under each other's identity.+ by_resource: dict[tuple[tuple[str, str], ...], list[TelemetryRecord]] = {}+ for record in batch:+ key = tuple(sorted(record.resource.items()))+ by_resource.setdefault(key, []).append(record)++ for records in by_resource.values():+ payload = build_logs_payload(+ [+ LogRecord(+ # Datadog renders this as the log message, and a blank+ # one makes the event unreadable there.+ body=record.name,+ severity_text="INFO",+ severity_number=_SEVERITY_INFO,+ time_nanos=record.timestamp_ns,+ attributes={"event.name": record.name, **record.properties},+ )+ for record in records+ ],+ _resource_from(records[0]),)+ _add_event_names(payload, [record.name for record in records])+ _add_relay_context(payload, records[0].resource)++ names = ", ".join(sorted({record.name for record in records}))+ try:+ await self._client.post("logs", payload)+ except OtlpPartialRejection as exc:+ logger.warning(+ "The relay rejected %d of %d telemetry event(s) in a batch "+ "(%s): %s. The rest were accepted; there is no retry.",+ exc.rejected,+ len(records),+ names,+ exc.reason,+ )+ except OtlpSendError as exc:+ # Switch has to keep working while analytics is down.+ logger.warning(+ "Telemetry batch of %d event(s) (%s) was not sent: %s. The "+ "batch is dropped; there is no retry.",+ len(records),+ names,+ exc,+ )+ if self._closing:+ # Feeding the rest to a relay that has just refused would+ # spend the shutdown budget losing them more slowly.+ self._refused_at_shutdown = Trueasync def aclose(self) -> None:- # `telemetry/setup.py` opens the client and `main._drain_telemetry`- # closes it, under a timeout shutdown depends on.- return None+ """Post everything still buffered, under the caller's timeout.++ `telemetry/setup.py` opens the HTTP client and `main._drain_telemetry`+ closes it, under a timeout shutdown depends on. The flusher is awaited+ with the rest, so that timeout cancels it along with this call rather+ than leaving it posting.+ """+ self._closing = True+ self._wake.set()+ try:+ self._take_buffer()+ posts: list[asyncio.Future[None]] = [+ asyncio.ensure_future(self._post_waiting())+ for _ in range(_SHUTDOWN_CONCURRENCY)+ ]+ if self._flusher is not None and not self._flusher.done():+ posts.append(self._flusher)+ await asyncio.gather(*posts)+ finally:+ self._report_unsent()++ def _report_unsent(self) -> None:+ """Say what shutdown could not post. Run from a `finally`: the caller's+ timeout cutting `aclose` off is the case that loses the most."""+ if unsent := self._posting + sum(len(batch) for batch in self._batches):+ logger.warning(+ "%d telemetry event(s) were never sent: %s.",+ unsent,+ "the relay refused a batch at shutdown, so the rest were not tried"+ if self._refused_at_shutdown+ else "shutdown ran out of time before they were posted",+ )+ if dropped := self._drops.take_pending():+ self._warn_dropped(dropped)+ if late := self._late.take_pending():+ logger.warning(+ "%d more telemetry event(s) arrived after shutdown began and "+ "were dropped.",+ late,+ )def _resource_from(record: TelemetryRecord) -> OtlpResource:@@ -146,10 +339,15 @@ def _add_relay_context(payload: dict[str, Any], resource: Mapping[str, str]) ->resource_log["resource"]["attributes"].extend(otlp_attributes(context))-def _add_event_name(payload: dict[str, Any], name: str) -> None:- """Set the record's own `eventName` field, which `build_logs_payload` does- not: an operational log line has no event name."""- for resource_log in payload["resourceLogs"]:- for scope_log in resource_log["scopeLogs"]:- for record in scope_log["logRecords"]:- record["eventName"] = name+def _add_event_names(payload: dict[str, Any], names: Sequence[str]) -> None:+ """Set each record's own `eventName` field, which `build_logs_payload` does+ not: an operational log line has no event name. `names` is in the order+ the records were handed to the encoder, which keeps it."""+ encoded = [+ record+ for resource_log in payload["resourceLogs"]+ for scope_log in resource_log["scopeLogs"]+ for record in scope_log["logRecords"]+ ]+ for record, name in zip(encoded, names, strict=True):+ record["eventName"] = namediff --git a/core/switch_core/telemetry/snapshot.py b/core/switch_core/telemetry/snapshot.pyindex 85238810e..998b6a116 100644--- a/core/switch_core/telemetry/snapshot.py+++ b/core/switch_core/telemetry/snapshot.py@@ -31,6 +31,7 @@ from sqlalchemy import (exists,func,select,+ union,union_all,)from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker@@ -131,6 +132,12 @@ class UsageCounts:chat_identity_count: int = 0chat_identity_active_1d: int = 0chat_identity_active_7d: int = 0+ # The other tiers, as chat identities like `chat_identity_active_*`:+ # anyone who said anything in any room, and anyone Switch has seen in a+ # room — see `_activity_tiers` for what that can and cannot see.+ chat_identity_posted_1d: int = 0+ chat_identity_posted_7d: int = 0+ chat_identity_in_room_count: int = 0room_count: int = 0room_agent_created_count: int = 0room_system_created_count: int = 0@@ -180,6 +187,9 @@ class UsageCounts:"chat_identity_count": self.chat_identity_count,"chat_identity_active_1d": self.chat_identity_active_1d,"chat_identity_active_7d": self.chat_identity_active_7d,+ "chat_identity_posted_1d": self.chat_identity_posted_1d,+ "chat_identity_posted_7d": self.chat_identity_posted_7d,+ "chat_identity_in_room_count": self.chat_identity_in_room_count,"room_count": self.room_count,"room_agent_created_count": self.room_agent_created_count,"room_system_created_count": self.room_system_created_count,@@ -387,6 +397,66 @@ def _active_humans(tenant_id: str, since: datetime) -> Select[tuple[str | None]])+def _activity_tiers(+ tenant_id: str, day_ago: datetime, week_ago: datetime+) -> Select[tuple[int, int, int]]:+ """Distinct human clients who said anything in any room in the last day+ and week, and those Switch has seen in a room: a member of a live room+ now, or anyone who posted in the last week.++ The posters are `_active_humans` without the agent condition: someone+ talking only to other people is not using an agent, but is not passive+ either. They are folded into the room count so the tiers nest: someone who+ posted this week and then left, or whose room was archived, is no longer a+ member of a live room but has still been seen. Rooms of every origin, like+ the activity counts: a person in an adopted channel is in Switch whether+ or not anybody created that room here.++ Not a channel's whole audience. A person gets a membership row only when+ they post, are added by name, or join after the channel was adopted;+ nothing records the members a channel already had, so someone who has+ only ever read is not here.++ One statement over one read of the week's messages, so the three come from+ the same snapshot and nest exactly, and the scan is not repeated per tier.+ """+ posters = (+ select(+ Message.sender_client_id.label("client_id"),+ func.max(Message.sent_at).label("last_posted_at"),+ )+ .join(Client, Client.id == Message.sender_client_id)+ .where(+ Message.tenant_id == tenant_id,+ Message.sent_at >= week_ago,+ *_human_message_conditions(tenant_id),+ )+ .group_by(Message.sender_client_id)+ .cte("posters")+ )+ members = (+ select(ClientRoom.client_id)+ .join(Client, Client.id == ClientRoom.client_id)+ .join(Room, Room.id == ClientRoom.room_id)+ .where(+ ClientRoom.tenant_id == tenant_id,+ Client.tenant_id == tenant_id,+ Client.type == HUMAN_CLIENT_TYPE,+ Room.tenant_id == tenant_id,+ Room.archived_at.is_(None),+ )+ )+ seen = union(members, select(posters.c.client_id)).subquery("seen")+ return select(+ select(func.count())+ .select_from(posters)+ .where(posters.c.last_posted_at >= day_ago)+ .scalar_subquery(),+ select(func.count()).select_from(posters).scalar_subquery(),+ select(func.count()).select_from(seen).scalar_subquery(),+ )++async def _count(session: AsyncSession, query: Select[Any]) -> int:result = await session.execute(select(func.count()).select_from(query.subquery()))return int(result.scalar_one())@@ -431,6 +501,12 @@ async def collect_tenant_counts(counts.chat_identity_active_7d += await _count(session, _active_humans(tenant_id, week_ago))+ posted_1d, posted_7d, in_room = (+ await session.execute(_activity_tiers(tenant_id, day_ago, week_ago))+ ).one()+ counts.chat_identity_posted_1d += posted_1d+ counts.chat_identity_posted_7d += posted_7d+ counts.chat_identity_in_room_count += in_roomcounts.room_active_1d += await _count(session, _human_interaction(tenant_id, day_ago))diff --git a/core/switch_core/transport/observer.py b/core/switch_core/transport/observer.pynew file mode 100644index 000000000..f7f2aefe7--- /dev/null+++ b/core/switch_core/transport/observer.py@@ -0,0 +1,48 @@+"""What the transport tells the rest of Switch about a message once it is sent.++The transport is the one place every participant's message passes through, so+it is where "a message was said in a room" can be observed without each writer+having to remember to report it. It knows nothing about who is listening: it+hands over the facts it already holds, after the commit, and carries on.+"""++from __future__ import annotations++from dataclasses import dataclass+from typing import Protocol+++@dataclass(frozen=True)+class ParticipantMessage:+ """A message a participant chose to say, as the transport saw it written.++ The same population the tenant is metered for: notices Switch posts on a+ participant's behalf are not in it. A multi-file post is reported once.+ """++ tenant_id: str+ room_id: str+ sender_client_id: str+ #: The writer's `ActorRole`: `human`, `agent`, `system` or `bridge`.+ sender_role: str+ has_attachment: bool+ in_thread: bool+++class ParticipantMessageObserver(Protocol):+ """Told about every participant message after it is committed.++ Called on the sender's path, so it must return at once and never raise:+ the message is already sent, and a failure here would tell the sender+ otherwise.+ """++ def observe(self, message: ParticipantMessage) -> None: ...+++class IgnoreParticipantMessages:+ """An observer that does nothing, for transports built where no one is+ listening — tooling and tests."""++ def observe(self, message: ParticipantMessage) -> None:+ return Nonediff --git a/core/switch_core/transport/postgres.py b/core/switch_core/transport/postgres.pyindex 0ca83ce15..95b5c1a64 100644--- a/core/switch_core/transport/postgres.py+++ b/core/switch_core/transport/postgres.py@@ -36,6 +36,7 @@ import uuidfrom datetime import UTC, datetimefrom typing import TYPE_CHECKING, Any+from switch_core.attachments import parse_attachment_groupfrom switch_core.db.models import (ClientRoom,MediaBlob,@@ -51,13 +52,19 @@ from switch_core.observability.catalogue import (DELIVERY_LAG,MESSAGES_DELIVERED,MESSAGES_SENT,+ OBSERVER_FAILURES,SEND_FAILURES,)from switch_core.observability.metrics import metrics+from switch_core.observability.throttle import WarningThrottlefrom switch_core.tenant_context import no_tenant, tenant_scopefrom switch_core.transport.content import media_content, message_contentfrom switch_core.transport.ephemeral import EphemeralBusfrom switch_core.transport.invites import InviteBus+from switch_core.transport.observer import (+ ParticipantMessage,+ ParticipantMessageObserver,+)from switch_core.transport.port import Handler, TransportHandlersfrom switch_core.transport.room_cache import DELIVERY_PAGEfrom switch_core.transport.stored import to_inbound@@ -99,6 +106,10 @@ _MEDIA_MSGTYPES = frozenset({"m.image", "m.file", "m.video", "m.audio"})# a tenant for them would bill it for how Switch works._METERED_KINDS = frozenset({"message", "media"})+# One for the process rather than one per transport: every client has a+# transport, and an observer that breaks breaks for all of them.+_observer_failures = WarningThrottle(60.0)+def _sent_kind(event_type: str, content: dict[str, object]) -> str:"""A bounded label for what was sent.@@ -175,6 +186,7 @@ class PostgresTransport:invites: InviteBus,ephemeral: EphemeralBus,room_cache: RoomDeliveryCache,+ message_observer: ParticipantMessageObserver,) -> None:self.user_id = user_idself.client_id = client_id@@ -209,6 +221,7 @@ class PostgresTransport:# When this client was last woken for each room, on the cache's clock.# A shared read only counts for this client if it started after this.self._woken: dict[str, int] = {}+ self._message_observer = message_observerself._handlers = TransportHandlers()# Per-room delivery position, and the transport-side id to hand back# to a handler. Both are keyed by the Switch room id, which is what@@ -800,8 +813,47 @@ class PostgresTransport:)raisemetrics().increment(MESSAGES_SENT, {"kind": kind, "actor": self._actor_role})+ if metered and kind in _METERED_KINDS:+ self._observe(tenant_id, room_id, kind, content)return result+ def _observe(+ self, tenant_id: str, room_id: str, kind: str, content: dict[str, object]+ ) -> None:+ """Tell the observer a participant said something.++ After the commit, so it never hears of a message that was rolled back.+ Guarded because the row is already committed: an exception leaving+ `_send` now would tell the sender that a message in the room was not+ sent, and its retry would post it twice. Counted, because a guard that+ only logs leaves nothing to alert on.+ """+ group = parse_attachment_group(content)+ if group is not None and group[1] != 0:+ # One post with several files is one message; its first part+ # stands for it.+ return+ try:+ self._message_observer.observe(+ ParticipantMessage(+ tenant_id=tenant_id,+ room_id=room_id,+ sender_client_id=self.client_id,+ sender_role=self._actor_role,+ has_attachment=kind == "media",+ in_thread=thread_root_of(content) is not None,+ )+ )+ except Exception:+ metrics().increment(OBSERVER_FAILURES, {"actor": self._actor_role})+ if (failures := _observer_failures.note()) is not None:+ logger.error(+ "Message observer failed; the message itself was sent. %d "+ "failure(s) since the last error.",+ failures,+ exc_info=True,+ )+async def set_typing(self, room_id: str, is_typing: bool) -> None:"""Not carried. Composing state is presence, and nothing consumes it."""diff --git a/core/tests/benchmarks/server.py b/core/tests/benchmarks/server.pyindex b40d90d78..3949daabb 100644--- a/core/tests/benchmarks/server.py+++ b/core/tests/benchmarks/server.py@@ -70,6 +70,9 @@ from switch_core.session_activity.listener import AgentSessionActivityListenerfrom switch_core.session_activity.outcomes import ApprovalOutcomesfrom switch_core.session_activity.service import AgentSessionActivityService, SwitchUserfrom switch_core.sessions.contract import ApprovalResult+from switch_core.telemetry.messages import MessageTelemetry+from switch_core.telemetry.service import TelemetryService+from switch_core.telemetry.sink import NullSinkfrom switch_core.tenant_context import bind_tenant_id, tenant_scopefrom switch_core.transport.ephemeral import EphemeralBusfrom switch_core.transport.invites import InviteBus@@ -487,6 +490,22 @@ async def _serve(),)+ # Telemetry off: these flows are not about reporting, but the+ # message hook must still be wired for the clients to build.+ message_telemetry = MessageTelemetry(+ telemetry=TelemetryService(+ sink=NullSink(),+ enabled=False,+ client_id="",+ service_name="switch-core",+ version=None,+ environment=None,+ telemetry_environment="dev",+ telemetry_internal=True,+ ),+ session_factory=session_factory,+ )+client_factory = ClientFactory(client_store=session_env.client_store,session_factory=session_factory,@@ -498,6 +517,7 @@ async def _serve(invites=invites,ephemeral=ephemeral,room_cache=room_cache,+ message_observer=message_telemetry,)client_factory.register("agent",@@ -515,6 +535,7 @@ async def _serve(hosted_launch_store=HostedLaunchStore(),connections=connections,frontend_base_url=config.frontend_base_url,+ message_telemetry=message_telemetry,)client_factory.register("user", HumanActor)client_factory.register("bridge", Actor)diff --git a/core/tests/integration/conftest.py b/core/tests/integration/conftest.pyindex e0572de00..22fb9e274 100644--- a/core/tests/integration/conftest.py+++ b/core/tests/integration/conftest.py@@ -85,6 +85,9 @@ from switch_core.messages.notify import MessageListenerfrom switch_core.provisioning import Provisioningfrom switch_core.provisioning.postgres import PostgresProvisioningfrom switch_core.room_service import RoomService+from switch_core.telemetry.messages import MessageTelemetry+from switch_core.telemetry.service import TelemetryService+from switch_core.telemetry.sink import NullSinkfrom switch_core.tenant_context import tenant_scopefrom switch_core.transport.ephemeral import EphemeralBusfrom switch_core.transport.invites import InviteBus@@ -567,6 +570,22 @@ async def harness(session_env: SessionEnv) -> AsyncIterator[Harness]:),)+ # Telemetry off: these flows are not about reporting, but the+ # message hook must still be wired for the clients to build.+ message_telemetry = MessageTelemetry(+ telemetry=TelemetryService(+ sink=NullSink(),+ enabled=False,+ client_id="",+ service_name="switch-core",+ version=None,+ environment=None,+ telemetry_environment="dev",+ telemetry_internal=True,+ ),+ session_factory=session_factory,+ )+client_factory = ClientFactory(client_store=session_env.client_store,session_factory=session_factory,@@ -579,6 +598,7 @@ async def harness(session_env: SessionEnv) -> AsyncIterator[Harness]:invites=invites,ephemeral=ephemeral,room_cache=room_cache,+ message_observer=message_telemetry,)client_factory.register("agent",@@ -596,6 +616,7 @@ async def harness(session_env: SessionEnv) -> AsyncIterator[Harness]:hosted_launch_store=HostedLaunchStore(),connections=connections,frontend_base_url=config.frontend_base_url,+ message_telemetry=message_telemetry,)client_factory.register("user", HumanActor)client_factory.register("bridge", Actor)diff --git a/core/tests/switch_core/bridges/agent/api/test_events_endpoint_dispatch.py b/core/tests/switch_core/bridges/agent/api/test_events_endpoint_dispatch.pyindex 23e7f5937..d979bc3e3 100644--- a/core/tests/switch_core/bridges/agent/api/test_events_endpoint_dispatch.py+++ b/core/tests/switch_core/bridges/agent/api/test_events_endpoint_dispatch.py@@ -444,7 +444,7 @@ class _RecordingSink:def __init__(self) -> None:self.sent: list[TelemetryRecord] = []- async def send(self, record: TelemetryRecord) -> None:+ def send(self, record: TelemetryRecord) -> None:self.sent.append(record)async def aclose(self) -> None:diff --git a/core/tests/switch_core/bridges/agent/protocol/test_controller_presence.py b/core/tests/switch_core/bridges/agent/protocol/test_controller_presence.pyindex 58e91387b..6a9710c78 100644--- a/core/tests/switch_core/bridges/agent/protocol/test_controller_presence.py+++ b/core/tests/switch_core/bridges/agent/protocol/test_controller_presence.py@@ -573,6 +573,7 @@ def _addressing_client(ns._fresh_agent = _fresh_agentns._gate_addressed = _gate_addressedns._note_hosted_addressed = _not_hosted+ ns._report_addressed = lambda *_a, **_k: Nonens._triggered_by_auto_reply = AgentConsumer._triggered_by_auto_replyns.actor = SimpleNamespace(send_message=sent)ns._event_buffer = SimpleNamespace(diff --git a/core/tests/switch_core/clients/test_agent_consumer_attachment_groups.py b/core/tests/switch_core/clients/test_agent_consumer_attachment_groups.pyindex b6306a901..1f8f041c4 100644--- a/core/tests/switch_core/clients/test_agent_consumer_attachment_groups.py+++ b/core/tests/switch_core/clients/test_agent_consumer_attachment_groups.py@@ -6,6 +6,8 @@ from types import SimpleNamespacefrom typing import Anyfrom unittest.mock import AsyncMock+import pytest+import switch_core.clients.agent_consumer as acfrom switch_core.clients.agent_consumer import AgentConsumer, _GateOutcomefrom switch_core.clients.room_meta import RoomMeta@@ -54,6 +56,15 @@ async def _session_factory(): # type: ignore[no-untyped-def]yield object()+class _FakeMessageTelemetry:+ def __init__(self) -> None:+ self.enabled = True+ self.addressed: list[dict[str, Any]] = []++ def agent_addressed(self, **kwargs: Any) -> None:+ self.addressed.append(kwargs)++class _FakeQueue:def __init__(self) -> None:self.events: list[Any] = []@@ -84,20 +95,30 @@ def _fake_client() -> SimpleNamespace:) -> _GateOutcome:return _GateOutcome(addressed=True, refusal=None)+ async def _is_available(_session: Any, _agent: Any, _room_id: str) -> bool:+ ns.availability_checks += 1+ return True+ns = SimpleNamespace(- agent=SimpleNamespace(id="agent-1", name="agent-a"),+ agent=SimpleNamespace(id="agent-1", name="agent-a", metadata_=None),+ tenant_id="tenant-1",session_factory=_session_factory,_event_buffer=queue,+ _message_telemetry=_FakeMessageTelemetry(),+ _triggered_by_auto_reply=AgentConsumer._triggered_by_auto_reply,_attachment_groups={},_attachment_group_timers={},_resolve_room_meta=_resolve_room_meta,_addressed=_addressed,_fresh_agent=_fresh_agent,_gate_addressed=_gate_addressed,+ _is_available=_is_available,+ availability_checks=0,queue=queue,_note_hosted_addressed=AsyncMock(return_value=None),)ns._emit_media = AgentConsumer._emit_media.__get__(ns)+ ns._report_addressed = AgentConsumer._report_addressed.__get__(ns)ns._schedule_attachment_group_flush = (AgentConsumer._schedule_attachment_group_flush.__get__(ns))@@ -440,3 +461,86 @@ class TestAddressingSurvivesCoalescing:)assert client.queue.events[0].payload.addressed is True+++async def test_a_grouped_post_addressed_to_the_agent_is_reported_once() -> None:+ """Three files in one post are one message asked of the agent, not three:+ the report follows the coalesced payload, not the parts."""+ client = _fake_client()+ for index in range(3):+ await AgentConsumer.on_media(+ client,+ _room(),+ _media_event(+ body="@agent-a look at these" if index == 0 else f"f{index}.png",+ event_id=f"$part-{index}",+ group={"id": "grp-r", "index": index, "total": 3},+ ),+ )++ assert client._message_telemetry.addressed == [+ {+ "tenant_id": "tenant-1",+ "room_id": "room-1",+ "sender_transport_user_id": "@alice:s",+ "from_platform": False,+ "agent_metadata": None,+ "agent_live": True,+ "has_attachment": True,+ }+ ]++[email protected]("late_index", [0, 2])+async def test_a_post_split_by_the_timeout_is_reported_once(late_index: int) -> None:+ """A group that times out incomplete is delivered again when the rest+ arrives. In a direct room both pieces address the agent, but it is one+ post: counted with its first part, as the transport counts it."""+ original = ac.ATTACHMENT_GROUP_TIMEOUT_SECONDS+ ac.ATTACHMENT_GROUP_TIMEOUT_SECONDS = 0.01+ try:+ client = _fake_client()++ async def _direct_room(_event: Any, _meta: RoomMeta) -> bool:+ return True++ client._addressed = _direct_room+ on_time = [index for index in range(3) if index != late_index]+ for batch in (on_time, [late_index]):+ for index in batch:+ await AgentConsumer.on_media(+ client,+ _room(),+ _media_event(+ body=f"f{index}.png",+ event_id=f"$part-{index}",+ group={"id": "grp-late", "index": index, "total": 3},+ ),+ )+ await asyncio.sleep(0.15)+ finally:+ ac.ATTACHMENT_GROUP_TIMEOUT_SECONDS = original++ assert len(client.queue.events) == 2+ assert len(client._message_telemetry.addressed) == 1+++async def test_media_skips_the_availability_check_while_telemetry_is_off() -> None:+ """The media path reads it only to report it, so off means no query."""+ client = _fake_client()+ client._message_telemetry.enabled = False+ await AgentConsumer.on_media(+ client, _room(), _media_event(body="@agent-a look", event_id="$solo")+ )++ assert client.availability_checks == 0+ assert len(client.queue.events) == 1+++async def test_media_that_does_not_address_the_agent_is_not_reported() -> None:+ client = _fake_client()+ await AgentConsumer.on_media(+ client, _room(), _media_event(body="just a picture", event_id="$solo")+ )++ assert client._message_telemetry.addressed == []diff --git a/core/tests/switch_core/clients/test_agent_consumer_pool_checkouts.py b/core/tests/switch_core/clients/test_agent_consumer_pool_checkouts.pyindex bc9a5cdad..b43fee430 100644--- a/core/tests/switch_core/clients/test_agent_consumer_pool_checkouts.py+++ b/core/tests/switch_core/clients/test_agent_consumer_pool_checkouts.py@@ -142,6 +142,9 @@ def _client(live_connection_ids=client._connections.live_connection_ids,)client._budget_guard = BudgetGuard(BudgetStore())+ client._message_telemetry = SimpleNamespace( # type: ignore[assignment]+ agent_addressed=lambda **_k: None+ )client._frontend_base_url = Noneclient._room_meta = {TRANSPORT_ROOM_ID: RoomMeta(diff --git a/core/tests/switch_core/clients/test_agent_consumer_reports_addressed.py b/core/tests/switch_core/clients/test_agent_consumer_reports_addressed.pynew file mode 100644index 000000000..9b265ac44--- /dev/null+++ b/core/tests/switch_core/clients/test_agent_consumer_reports_addressed.py@@ -0,0 +1,289 @@+"""An agent's consumer reports each message its agent was actually asked to act on.++"Asked" means addressed *and* let through: a message the addressing policy or a+budget turned away was not a request the agent received, and Switch's own+auto-replies are not anyone asking for anything. Whether the agent was there+to take it is reported alongside, not used to leave the message out.+"""++from __future__ import annotations++from contextlib import asynccontextmanager+from types import SimpleNamespace+from typing import Any++import pytest++from switch_core.clients import agent_consumer as agent_consumer_module+from switch_core.clients.admin_messages import PLATFORM_MARKER+from switch_core.clients.agent_consumer import (+ AUTO_REPLY_FLAG,+ AgentConsumer,+ HostedNote,+ _GateOutcome,+)+from switch_core.clients.room_meta import RoomMeta+from switch_core.observability.metrics import MetricsRegistry, install, uninstall+from switch_core.observability.throttle import WarningThrottle+from switch_core.transport import InboundMessage, RoomRef+++@asynccontextmanager+async def _session_factory(): # type: ignore[no-untyped-def]+ yield object()+++class _FakeMessageTelemetry:+ def __init__(self) -> None:+ self.addressed: list[dict[str, Any]] = []++ def agent_addressed(self, **kwargs: Any) -> None:+ self.addressed.append(kwargs)+++class _BrokenMessageTelemetry:+ def agent_addressed(self, **kwargs: Any) -> None:+ raise RuntimeError("telemetry bug")+++def _consumer(+ *,+ addressed: bool,+ allowed: bool = True,+ live: bool = True,+ hosted: HostedNote | None = None,+) -> SimpleNamespace:+ """An AgentConsumer stand-in with addressing, the policy gate, whether the+ agent is live in the room and a hosted agent's mailbox fixed by the test."""+ meta = RoomMeta(+ room_id="room-1", name="Room", bridge_id=None, channel_type="channel_public"+ )++ async def _resolve_room_meta(_transport_room_id: str) -> RoomMeta:+ return meta++ async def _compute_addressed(_s: object, _e: object, _m: object) -> bool:+ return addressed++ async def _fresh_agent(_s: object) -> object:+ return ns.agent++ async def _gate_addressed(+ _s: object, _a: object, _e: object, _m: object+ ) -> _GateOutcome:+ return _GateOutcome(addressed=allowed, refusal=None)++ async def _is_available(_s: object, _a: object, _r: str) -> bool:+ return live++ async def _reply_when_unavailable_here(+ _s: object, _a: object, _m: object, _h: str+ ) -> str | None:+ return None++ async def _note_hosted_addressed(_a: object, _e: object) -> HostedNote | None:+ return hosted++ ns = SimpleNamespace(+ agent=SimpleNamespace(+ id="agent-1", name="agent-a", metadata_={"known_agent_type": "codex"}+ ),+ tenant_id="tenant-1",+ session_factory=_session_factory,+ _resolve_room_meta=_resolve_room_meta,+ _addressed_without_lookup=lambda _e, _m: None,+ _compute_addressed=_compute_addressed,+ _fresh_agent=_fresh_agent,+ _gate_addressed=_gate_addressed,+ _is_available=_is_available,+ _reply_when_unavailable_here=_reply_when_unavailable_here,+ _note_hosted_addressed=_note_hosted_addressed,+ _post_auto_reply=_ignore_post,+ _sender_handle=lambda _e: "@alice",+ _triggered_by_auto_reply=AgentConsumer._triggered_by_auto_reply,+ _event_buffer=SimpleNamespace(enqueue=lambda *a, **k: ns.enqueued.append(a)),+ enqueued=[],+ _message_telemetry=_FakeMessageTelemetry(),+ )+ ns._report_addressed = AgentConsumer._report_addressed.__get__(ns)+ return ns+++async def _ignore_post(*_args: object, **_kwargs: object) -> None:+ return None+++def _message(**content: object) -> InboundMessage:+ return InboundMessage(+ room_id="!room:s",+ event_id="$evt",+ sender="@alice:s",+ timestamp=0,+ content={"sender_name": "alice", **content},+ body="@agent-a can you help",+ sender_name="alice",+ )+++async def _deliver(consumer: SimpleNamespace, event: InboundMessage) -> None:+ await AgentConsumer.on_message(consumer, RoomRef(room_id="!room:s"), event) # type: ignore[arg-type]+++async def test_an_addressed_message_is_reported_once() -> None:+ consumer = _consumer(addressed=True)++ await _deliver(consumer, _message())++ assert consumer._message_telemetry.addressed == [+ {+ "tenant_id": "tenant-1",+ "room_id": "room-1",+ "sender_transport_user_id": "@alice:s",+ "from_platform": False,+ "agent_metadata": {"known_agent_type": "codex"},+ "agent_live": True,+ "has_attachment": False,+ }+ ]+++async def test_room_chatter_is_not_reported() -> None:+ consumer = _consumer(addressed=False)++ await _deliver(consumer, _message())++ assert consumer._message_telemetry.addressed == []+++async def test_a_message_the_policy_turned_away_is_not_reported() -> None:+ consumer = _consumer(addressed=True, allowed=False)++ await _deliver(consumer, _message())++ assert consumer._message_telemetry.addressed == []+++async def test_an_auto_reply_is_not_reported() -> None:+ consumer = _consumer(addressed=True)++ await _deliver(consumer, _message(**{AUTO_REPLY_FLAG: True}))++ assert consumer._message_telemetry.addressed == []+++async def test_a_platform_message_says_it_came_from_the_platform() -> None:+ consumer = _consumer(addressed=True)++ await _deliver(consumer, _message(**{PLATFORM_MARKER: {}}))++ [report] = consumer._message_telemetry.addressed+ assert report["from_platform"] is True+++async def test_a_message_to_an_offline_agent_is_reported_as_not_live() -> None:+ consumer = _consumer(addressed=True, live=False)++ await _deliver(consumer, _message())++ [report] = consumer._message_telemetry.addressed+ assert report["agent_live"] is False+++async def test_a_message_a_hosted_agents_mailbox_refused_is_reported_as_not_live() -> (+ None+):+ """A stopped or broken cloud worker is an agent that was asked and was not+ there, the same as any other offline agent — not a message nobody sent."""+ consumer = _consumer(+ addressed=True,+ live=False,+ hosted=HostedNote(+ launch=None, machine=None, refusal="worker stopped", deliver=False+ ),+ )++ await _deliver(consumer, _message())++ [report] = consumer._message_telemetry.addressed+ assert report["agent_live"] is False+++async def test_a_message_a_live_hosted_worker_could_not_take_is_not_live() -> None:+ """A full mailbox or a lost provider refuses the message even with a+ worker attached, so the agent could not act on it."""+ consumer = _consumer(+ addressed=True,+ live=True,+ hosted=HostedNote(+ launch=None, machine=None, refusal="mailbox full", deliver=False+ ),+ )++ await _deliver(consumer, _message())++ [report] = consumer._message_telemetry.addressed+ assert report["agent_live"] is False+++async def test_a_message_held_for_a_waking_hosted_agent_is_reported() -> None:+ consumer = _consumer(+ addressed=True,+ live=False,+ hosted=HostedNote(launch=None, machine=None, refusal=None, deliver=True),+ )++ await _deliver(consumer, _message())++ assert len(consumer._message_telemetry.addressed) == 1+++async def test_a_redelivery_the_mailbox_already_holds_is_not_counted_again() -> None:+ """The same message seen a second time is one request, not two."""+ consumer = _consumer(+ addressed=True,+ hosted=HostedNote(launch=None, machine=None, refusal=None, deliver=False),+ )++ await _deliver(consumer, _message())++ assert consumer._message_telemetry.addressed == []+++async def test_a_telemetry_failure_does_not_cost_the_agent_the_message(+ monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture+) -> None:+ """Caught, so it is counted, and logged once rather than once per+ message."""+ consumer = _consumer(addressed=True)+ consumer._message_telemetry = _BrokenMessageTelemetry()+ registry = MetricsRegistry()+ install(registry)+ monkeypatch.setattr(+ agent_consumer_module, "_report_failures", WarningThrottle(60.0)+ )++ try:+ with caplog.at_level("ERROR"):+ await _deliver(consumer, _message())+ await _deliver(consumer, _message())+ failures = next(+ p+ for p in registry.collect()+ if p.name == "switch.messages.observer_failures"+ )+ finally:+ uninstall()++ assert len(consumer.enqueued) == 2+ assert failures.numbers[0].value == 2.0+ assert dict(failures.numbers[0].attributes) == {"actor": "agent"}+ assert (+ len(+ [+ r+ for r in caplog.records+ if "Could not report a message addressed" in r.getMessage()+ ]+ )+ == 1+ )diff --git a/core/tests/switch_core/clients/test_client_tenant_plumbing.py b/core/tests/switch_core/clients/test_client_tenant_plumbing.pyindex d58d88d0b..18e2eae70 100644--- a/core/tests/switch_core/clients/test_client_tenant_plumbing.py+++ b/core/tests/switch_core/clients/test_client_tenant_plumbing.py@@ -51,6 +51,7 @@ from switch_core.db.stores.usage_store import UsageStorefrom switch_core.tenant_context import tenant_scopefrom switch_core.transport.ephemeral import EphemeralBusfrom switch_core.transport.invites import InviteBus+from switch_core.transport.observer import IgnoreParticipantMessagesdef _factory(session_factory: async_sessionmaker[AsyncSession]) -> ClientFactory:@@ -66,6 +67,7 @@ def _factory(session_factory: async_sessionmaker[AsyncSession]) -> ClientFactoryinvites=InviteBus(),ephemeral=EphemeralBus(),room_cache=MagicMock(),+ message_observer=IgnoreParticipantMessages(),)factory.register("user", Actor)return factorydiff --git a/core/tests/switch_core/clients/test_no_session_reply_threading.py b/core/tests/switch_core/clients/test_no_session_reply_threading.pyindex 63139edb6..dcbb6149b 100644--- a/core/tests/switch_core/clients/test_no_session_reply_threading.py+++ b/core/tests/switch_core/clients/test_no_session_reply_threading.py@@ -99,6 +99,7 @@ def _fake_self(_triggered_by_auto_reply=AgentConsumer._triggered_by_auto_reply,actor=SimpleNamespace(send_message=send_message),_event_buffer=SimpleNamespace(enqueue=lambda *a, **k: None),+ _report_addressed=lambda *a, **k: None,)# Exercise the real sender-tagging and auto-reply helpers.ns._sender_handle = AgentConsumer._sender_handle.__get__(ns)diff --git a/core/tests/switch_core/gateway/test_hosted_mailbox.py b/core/tests/switch_core/gateway/test_hosted_mailbox.pyindex 39ea47219..836eed41d 100644--- a/core/tests/switch_core/gateway/test_hosted_mailbox.py+++ b/core/tests/switch_core/gateway/test_hosted_mailbox.py@@ -802,6 +802,7 @@ async def agent_client(app) -> SimpleNamespace:)posted: list[str] = []enqueued: list[AgentEvent] = []+ reported: list[dict[str, Any]] = []async def send_message(_room_id: str, body: str, **_kwargs: Any) -> str:posted.append(body)@@ -847,13 +848,19 @@ async def agent_client(app) -> SimpleNamespace:_waking_notice_revisions={},_unreachable_notice_revisions={},actor=SimpleNamespace(send_message=send_message),+ _message_telemetry=SimpleNamespace(+ enabled=True,+ agent_addressed=lambda **kwargs: reported.append(kwargs),+ ),posted=posted,enqueued=enqueued,+ reported=reported,)for name in ("_note_hosted_addressed","_emit_media","_post_auto_reply",+ "_report_addressed","_sender_handle",):setattr(client, name, getattr(AgentConsumer, name).__get__(client))@@ -893,6 +900,9 @@ async def test_media_to_an_errored_running_machine_is_refused_not_queued(assert await rows(app) == {}assert client.posted == [f"@someone {_HOSTED_MACHINE_ERROR_MESSAGE}"]assert client.enqueued == []+ # Still a message the agent was asked, by someone it could not answer.+ [report] = client.reported+ assert report["agent_live"] is Falseasync def test_task_delegate_to_an_errored_running_machine_is_refused(mailbox_app):@@ -935,6 +945,30 @@ async def test_mention_to_an_errored_running_machine_is_answered_once(mailbox_apassert await rows(app) == {}assert client.posted == [f"@someone {_HOSTED_MACHINE_ERROR_MESSAGE}"]assert client.enqueued == []+ [report] = client.reported+ assert report["agent_live"] is False+++async def test_a_mention_the_mailbox_takes_is_a_message_the_agent_received(+ mailbox_app,+):+ """Held for the worker rather than refused: asked, while not yet live."""+ app = mailbox_app+ client = await agent_client(app)+ message = InboundMessage(+ room_id="!room:example.com",+ event_id="$m1",+ sender="@someone:example.com",+ timestamp=1700000000000,+ content={"sender_name": "someone"},+ body="@agent hello",+ sender_name="someone",+ )+ await AgentConsumer.on_message(+ client, RoomRef(room_id="!room:example.com"), message+ ) # type: ignore[arg-type]+ [report] = client.reported+ assert report["agent_live"] is False@pytest.mark.parametrize(diff --git a/core/tests/switch_core/observability/test_otlp.py b/core/tests/switch_core/observability/test_otlp.pyindex 86671a6da..45f8676e4 100644--- a/core/tests/switch_core/observability/test_otlp.py+++ b/core/tests/switch_core/observability/test_otlp.py@@ -18,6 +18,7 @@ from switch_core.observability.otlp import (MetricPayload,NumberPoint,OtlpClient,+ OtlpPartialRejection,OtlpResource,OtlpSendError,build_logs_payload,@@ -292,6 +293,19 @@ async def test_partial_rejection_inside_a_200_is_raised():await client.post("metrics", {})+async def test_a_partial_rejection_says_how_many_records_and_why():+ """A batching caller reports what it lost from this, not the whole batch."""+ body = {"partialSuccess": {"rejectedLogRecords": "2", "errorMessage": "too big"}}+ client = _client(lambda request: httpx.Response(200, json=body))++ with pytest.raises(OtlpPartialRejection) as raised:+ await client.post("logs", {})++ assert raised.value.rejected == 2+ assert raised.value.reason == "too big"++@pytest.mark.asyncioasync def test_empty_partial_success_is_not_a_failure():client = _client(lambda request: httpx.Response(200, json={"partialSuccess": {}}))diff --git a/core/tests/switch_core/observability/test_throttle.py b/core/tests/switch_core/observability/test_throttle.pynew file mode 100644index 000000000..54e72b1b3--- /dev/null+++ b/core/tests/switch_core/observability/test_throttle.py@@ -0,0 +1,55 @@+"""The rate limit on warnings that could otherwise fire once per message."""++from __future__ import annotations++from types import SimpleNamespace++import pytest++from switch_core.observability import throttle+from switch_core.observability.throttle import WarningThrottle+++class _Clock:+ def __init__(self, now: float) -> None:+ self.now = now++ def __call__(self) -> float:+ return self.now+++def clock(monkeypatch: pytest.MonkeyPatch) -> _Clock:+ fake = _Clock(5.0)+ monkeypatch.setattr(throttle, "time", SimpleNamespace(monotonic=fake))+ return fake+++def test_the_first_occurrence_logs_even_just_after_boot(clock: _Clock) -> None:+ """The monotonic clock counts from boot, so a "last logged at 0" start+ would swallow every warning for the first minute of a fresh host."""+ assert WarningThrottle(60.0).note() == 1+++def test_occurrences_inside_the_interval_are_counted_not_logged(+ clock: _Clock,+) -> None:+ warnings = WarningThrottle(60.0)+ warnings.note()++ clock.now += 30+ assert warnings.note() is None+ assert warnings.note() is None++ clock.now += 31+ assert warnings.note() == 3+++def test_what_was_never_logged_can_be_taken_at_the_end(clock: _Clock) -> None:+ warnings = WarningThrottle(60.0)+ warnings.note()+ warnings.note()+ warnings.note()++ assert warnings.take_pending() == 2+ assert warnings.take_pending() == 0diff --git a/core/tests/switch_core/session_activity/test_session_started_route.py b/core/tests/switch_core/session_activity/test_session_started_route.pyindex ff55dfbd1..b097ba00f 100644--- a/core/tests/switch_core/session_activity/test_session_started_route.py+++ b/core/tests/switch_core/session_activity/test_session_started_route.py@@ -40,7 +40,7 @@ class _RecordingSink:def __init__(self) -> None:self.sent: list[TelemetryRecord] = []- async def send(self, record: TelemetryRecord) -> None:+ def send(self, record: TelemetryRecord) -> None:self.sent.append(record)async def aclose(self) -> None:diff --git a/core/tests/switch_core/telemetry/test_bridge_connect_reporting.py b/core/tests/switch_core/telemetry/test_bridge_connect_reporting.pyindex 9003eeef1..09324f3fe 100644--- a/core/tests/switch_core/telemetry/test_bridge_connect_reporting.py+++ b/core/tests/switch_core/telemetry/test_bridge_connect_reporting.py@@ -57,7 +57,7 @@ class _RecordingSink:def __init__(self) -> None:self.sent: list[TelemetryRecord] = []- async def send(self, record: TelemetryRecord) -> None:+ def send(self, record: TelemetryRecord) -> None:self.sent.append(record)async def aclose(self) -> None:diff --git a/core/tests/switch_core/telemetry/test_catalogue.py b/core/tests/switch_core/telemetry/test_catalogue.pyindex 15a1354de..950f19b4a 100644--- a/core/tests/switch_core/telemetry/test_catalogue.py+++ b/core/tests/switch_core/telemetry/test_catalogue.py@@ -177,3 +177,23 @@ class TestNaming:keeps `room_created` here apart from the Console's."""assert wire_name("room_created") == f"{EVENT_NAME_PREFIX}.room_created"assert EVENT_NAME_PREFIX == "switch_core"+++class TestAgentRuntimes:+ """`unknown` is a lookup that failed, so only the event that looks the+ agent up may carry it. Everywhere else the runtime comes from an agent in+ hand, and `unknown` there would be a bug hiding as a value."""++ def test_the_sent_event_may_say_its_runtime_is_unknown(self) -> None:+ payload = _valid_payload("agent_message_sent")+ payload["known_agent_type"] = "unknown"+ validate("agent_message_sent", payload) # type: ignore[arg-type]++ @pytest.mark.parametrize(+ "event", ["agent_message_received", "first_agent_registered"]+ )+ def test_events_with_the_agent_in_hand_may_not(self, event: str) -> None:+ payload = _valid_payload(event)+ payload["known_agent_type"] = "unknown"+ with pytest.raises(TelemetryCatalogueError, match="expected one of"):+ validate(event, payload) # type: ignore[arg-type]diff --git a/core/tests/switch_core/telemetry/test_deployment_and_reporter.py b/core/tests/switch_core/telemetry/test_deployment_and_reporter.pyindex e85d01689..2acfd7639 100644--- a/core/tests/switch_core/telemetry/test_deployment_and_reporter.py+++ b/core/tests/switch_core/telemetry/test_deployment_and_reporter.py@@ -56,7 +56,7 @@ class _RecordingSink:def __init__(self) -> None:self.sent: list[TelemetryRecord] = []- async def send(self, record: TelemetryRecord) -> None:+ def send(self, record: TelemetryRecord) -> None:self.sent.append(record)async def aclose(self) -> None:diff --git a/core/tests/switch_core/telemetry/test_first_room_active.py b/core/tests/switch_core/telemetry/test_first_room_active.pyindex 5fa1af07c..9b5690512 100644--- a/core/tests/switch_core/telemetry/test_first_room_active.py+++ b/core/tests/switch_core/telemetry/test_first_room_active.py@@ -30,7 +30,7 @@ class _RecordingSink:def __init__(self) -> None:self.sent: list[TelemetryRecord] = []- async def send(self, record: TelemetryRecord) -> None:+ def send(self, record: TelemetryRecord) -> None:self.sent.append(record)async def aclose(self) -> None:diff --git a/core/tests/switch_core/telemetry/test_merge_fixes.py b/core/tests/switch_core/telemetry/test_merge_fixes.pyindex 40295e17e..7264f77af 100644--- a/core/tests/switch_core/telemetry/test_merge_fixes.py+++ b/core/tests/switch_core/telemetry/test_merge_fixes.py@@ -36,7 +36,7 @@ class _RecordingSink:def __init__(self) -> None:self.sent: list[TelemetryRecord] = []- async def send(self, record: TelemetryRecord) -> None:+ def send(self, record: TelemetryRecord) -> None:self.sent.append(record)async def aclose(self) -> None:diff --git a/core/tests/switch_core/telemetry/test_message_events.py b/core/tests/switch_core/telemetry/test_message_events.pynew file mode 100644index 000000000..d188da64b--- /dev/null+++ b/core/tests/switch_core/telemetry/test_message_events.py@@ -0,0 +1,1050 @@+"""The per-message events, against real Postgres.++What these pin is what each event says about the room and the sender — the+platform, the room's size, who spoke — because those are read from the+database by the worker rather than handed over by the sender, and a wrong join+there would be a wrong chart with no error anywhere.+"""++from __future__ import annotations++import asyncio+import time+import uuid+from types import SimpleNamespace++import pytest+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker++from switch_core.db.models import (+ Agent,+ ApiKey,+ Client,+ ClientRoom,+ CollaborationBridge,+ Room,+ User,+)+from switch_core.telemetry import messages as messages_module+from switch_core.telemetry.messages import MessageTelemetry, _RoomFacts, _TtlCache+from switch_core.telemetry.service import TelemetryService+from switch_core.telemetry.sink import TelemetryRecord+from switch_core.transport.observer import ParticipantMessage++TENANT_ZERO = "00000000-0000-0000-0000-000000000000"+++class _RecordingSink:+ def __init__(self) -> None:+ self.sent: list[TelemetryRecord] = []++ def send(self, record: TelemetryRecord) -> None:+ self.sent.append(record)++ async def aclose(self) -> None:+ return None+++def _service(sink: _RecordingSink, *, enabled: bool = True) -> TelemetryService:+ return TelemetryService(+ sink=sink,+ enabled=enabled,+ client_id="deployment-uuid",+ service_name="switch-core",+ version="1.0.0",+ environment=None,+ telemetry_environment="prod",+ telemetry_internal=False,+ )+++async def _client(session: AsyncSession, client_type: str) -> Client:+ client = Client(+ transport_user_id=f"@{client_type}-{uuid.uuid4().hex[:8]}:test",+ display_name=f"{client_type} client",+ type=client_type,+ )+ session.add(client)+ await session.flush()+ return client+++async def _agent(session: AsyncSession, runtime: str) -> Agent:+ slug = uuid.uuid4().hex[:10]+ owner = User(+ name="owner", email=f"owner-{slug}@test", role="user", password_hash="x"+ )+ session.add(owner)+ await session.flush()+ key = ApiKey(+ user_id=owner.id,+ key_hash=f"hash-{slug}",+ encrypted_key="enc",+ label="test",+ type="agent",+ )+ backing = await _client(session, "agent")+ session.add(key)+ await session.flush()+ agent = Agent(+ name=f"agent-{slug[:6]}",+ description="d",+ agent_type="session_addressable",+ connector_type="http",+ integration_profile={},+ client_id=backing.id,+ api_key_id=key.id,+ metadata_={"known_agent_type": runtime},+ )+ session.add(agent)+ await session.flush()+ return agent+++async def _room(+ session: AsyncSession, *, bridge_platform: str | None, channel_type: str+) -> Room:+ bridge_id: str | None = None+ if bridge_platform is not None:+ bridge = CollaborationBridge(+ type=bridge_platform,+ display_name="workspace",+ client_id=(await _client(session, "bridge")).id,+ status="active",+ )+ session.add(bridge)+ await session.flush()+ bridge_id = bridge.id+ room = Room(+ transport_room_id=f"!{uuid.uuid4().hex[:10]}:test",+ name="a room",+ description="",+ channel_type=channel_type,+ bridge_id=bridge_id,+ )+ session.add(room)+ await session.flush()+ return room+++async def _join(session: AsyncSession, client_id: str, room: Room) -> None:+ session.add(ClientRoom(client_id=client_id, room_id=room.id))+ await session.flush()+++class _World:+ """A Slack-bridged private room with two people, one Codex agent, and the+ bridge's own member — which is neither, and must count as neither."""++ room: Room+ humans: list[Client]+ agent: Agent+++async def _world(session_factory: async_sessionmaker[AsyncSession]) -> _World:+ world = _World()+ async with session_factory() as session:+ world.room = await _room(+ session, bridge_platform="slack", channel_type="channel_private"+ )+ world.humans = [await _client(session, "user") for _ in range(2)]+ world.agent = await _agent(session, "codex")+ for human in world.humans:+ await _join(session, human.id, world.room)+ await _join(session, world.agent.client_id, world.room)+ await _join(session, (await _client(session, "bridge")).id, world.room)+ await session.commit()+ return world+++def _said(world: _World, sender_client_id: str, role: str) -> ParticipantMessage:+ return ParticipantMessage(+ tenant_id=TENANT_ZERO,+ room_id=world.room.id,+ sender_client_id=sender_client_id,+ sender_role=role,+ has_attachment=False,+ in_thread=False,+ )+++def _said_in(+ room_id: str, *, sender_role: str = "human", sender_client_id: str = "someone"+) -> ParticipantMessage:+ return ParticipantMessage(+ tenant_id=TENANT_ZERO,+ room_id=room_id,+ sender_client_id=sender_client_id,+ sender_role=sender_role,+ has_attachment=False,+ in_thread=False,+ )+++async def _a_slack_room(tenant_id: str, room_id: str) -> _RoomFacts:+ return _RoomFacts(+ bridge_platform="slack",+ channel_type="channel_public",+ user_count=1,+ agent_count=1,+ )+++class _Unreachable:+ """A session factory for a database that will not answer."""++ def __init__(self) -> None:+ self.attempts = 0++ def __call__(self) -> object:+ self.attempts += 1+ raise ConnectionError("database unreachable")+++async def _reported(+ messages: MessageTelemetry, service: TelemetryService, sink: _RecordingSink+) -> list[tuple[str, dict[str, object]]]:+ await messages.aclose()+ await service.aclose()+ return [(record.name, record.properties) for record in sink.sent]+++class TestRoomMessageSent:+ async def test_a_person_speaking_is_reported_with_the_room_it_was_said_in(+ self, session_factory: async_sessionmaker[AsyncSession]+ ) -> None:+ world = await _world(session_factory)+ sink = _RecordingSink()+ service = _service(sink)+ messages = MessageTelemetry(telemetry=service, session_factory=session_factory)++ messages.observe(_said(world, world.humans[0].id, "human"))++ assert await _reported(messages, service, sink) == [+ (+ "switch_core.room_message_sent",+ {+ "sender_kind": "user",+ "bridge_platform": "slack",+ "channel_type": "channel_private",+ "room_user_count": 2,+ "room_agent_count": 1,+ "has_attachment": False,+ "in_thread": False,+ },+ )+ ]++ async def test_an_internal_room_has_no_platform(+ self, session_factory: async_sessionmaker[AsyncSession]+ ) -> None:+ async with session_factory() as session:+ room = await _room(session, bridge_platform=None, channel_type="direct")+ human = await _client(session, "user")+ await _join(session, human.id, room)+ await session.commit()+ sink = _RecordingSink()+ service = _service(sink)+ messages = MessageTelemetry(telemetry=service, session_factory=session_factory)++ messages.observe(+ ParticipantMessage(+ tenant_id=TENANT_ZERO,+ room_id=room.id,+ sender_client_id=human.id,+ sender_role="human",+ has_attachment=True,+ in_thread=True,+ )+ )++ [(_, properties)] = await _reported(messages, service, sink)+ assert properties["bridge_platform"] == "none"+ assert properties["channel_type"] == "direct"+ assert properties["has_attachment"] is True+ assert properties["in_thread"] is True++ async def test_a_room_that_cannot_be_found_is_reported_as_unknown(+ self, session_factory: async_sessionmaker[AsyncSession]+ ) -> None:+ # Still counted: a lookup problem must read as `unknown` in the chart,+ # not as fewer messages.+ sink = _RecordingSink()+ service = _service(sink)+ messages = MessageTelemetry(telemetry=service, session_factory=session_factory)++ messages.observe(+ ParticipantMessage(+ tenant_id=TENANT_ZERO,+ room_id=str(uuid.uuid4()),+ sender_client_id="whoever",+ sender_role="human",+ has_attachment=False,+ in_thread=False,+ )+ )++ [(_, properties)] = await _reported(messages, service, sink)+ assert properties["bridge_platform"] == "unknown"+ assert properties["channel_type"] == "unknown"+ assert properties["room_user_count"] == -1+ assert properties["room_agent_count"] == -1+++class TestAgentMessageSent:+ async def test_an_agent_speaking_is_also_reported_with_its_runtime(+ self, session_factory: async_sessionmaker[AsyncSession]+ ) -> None:+ world = await _world(session_factory)+ sink = _RecordingSink()+ service = _service(sink)+ messages = MessageTelemetry(telemetry=service, session_factory=session_factory)++ messages.observe(_said(world, world.agent.client_id, "agent"))++ reported = await _reported(messages, service, sink)+ assert [name for name, _ in reported] == [+ "switch_core.room_message_sent",+ "switch_core.agent_message_sent",+ ]+ assert reported[0][1]["sender_kind"] == "agent"+ assert reported[1][1] == {+ "known_agent_type": "codex",+ "bridge_platform": "slack",+ "channel_type": "channel_private",+ "room_user_count": 2,+ "has_attachment": False,+ "in_thread": False,+ }++ async def test_an_agent_that_cannot_be_found_has_an_unknown_runtime(+ self, session_factory: async_sessionmaker[AsyncSession]+ ) -> None:+ """Not `none`, which is a real answer — an agent that declares no+ runtime. A lookup problem must not read as a population of those."""+ world = await _world(session_factory)+ async with session_factory() as session:+ unregistered = await _client(session, "agent")+ await session.commit()+ sink = _RecordingSink()+ service = _service(sink)+ messages = MessageTelemetry(telemetry=service, session_factory=session_factory)++ messages.observe(_said(world, unregistered.id, "agent"))++ reported = await _reported(messages, service, sink)+ assert reported[1][1]["known_agent_type"] == "unknown"+++class TestAgentMessageReceived:+ async def test_a_person_asking_an_agent_is_reported(+ self, session_factory: async_sessionmaker[AsyncSession]+ ) -> None:+ world = await _world(session_factory)+ sink = _RecordingSink()+ service = _service(sink)+ messages = MessageTelemetry(telemetry=service, session_factory=session_factory)++ messages.agent_addressed(+ tenant_id=TENANT_ZERO,+ room_id=world.room.id,+ sender_transport_user_id=world.humans[0].transport_user_id,+ from_platform=False,+ agent_metadata={"known_agent_type": "claude-code"},+ agent_live=True,+ has_attachment=False,+ )++ assert await _reported(messages, service, sink) == [+ (+ "switch_core.agent_message_received",+ {+ "sender_kind": "user",+ "known_agent_type": "claude-code",+ "bridge_platform": "slack",+ "channel_type": "channel_private",+ "has_attachment": False,+ "agent_live": True,+ },+ )+ ]++ async def test_who_asked_is_told_apart(+ self, session_factory: async_sessionmaker[AsyncSession]+ ) -> None:+ world = await _world(session_factory)+ sink = _RecordingSink()+ service = _service(sink)+ messages = MessageTelemetry(telemetry=service, session_factory=session_factory)+ async with session_factory() as session:+ sender_agent = await _client(session, "agent")+ await session.commit()++ for sender, from_platform in (+ (sender_agent.transport_user_id, False),+ ("@admin:test", True),+ ("@nobody-we-know:test", False),+ ):+ messages.agent_addressed(+ tenant_id=TENANT_ZERO,+ room_id=world.room.id,+ sender_transport_user_id=sender,+ from_platform=from_platform,+ agent_metadata=None,+ agent_live=False,+ has_attachment=False,+ )++ reported = await _reported(messages, service, sink)+ assert [p["sender_kind"] for _, p in reported] == [+ "agent",+ "platform",+ "unknown",+ ]+ assert {p["known_agent_type"] for _, p in reported} == {"none"}+++class TestFailingLookups:+ async def test_a_room_whose_lookup_failed_is_not_retried_and_warns_once(+ self,+ session_factory: async_sessionmaker[AsyncSession],+ caplog: pytest.LogCaptureFixture,+ ) -> None:+ """A busy room whose lookup fails must not cost a query, and a+ traceback, for every message said in it."""+ world = await _world(session_factory)+ sink = _RecordingSink()+ service = _service(sink)+ messages = MessageTelemetry(telemetry=service, session_factory=session_factory)+ attempts = 0++ async def _broken(tenant_id: str, room_id: str) -> None:+ nonlocal attempts+ attempts += 1+ raise ConnectionError("database unreachable")++ messages._load_room_facts = _broken # type: ignore[method-assign]++ with caplog.at_level("WARNING"):+ for _ in range(3):+ messages.observe(_said(world, world.humans[0].id, "human"))+ reported = await _reported(messages, service, sink)++ assert attempts == 1+ assert [p["bridge_platform"] for _, p in reported] == ["unknown"] * 3+ assert (+ len([r for r in caplog.records if "could not look up" in r.getMessage()])+ == 1+ )++ async def test_one_failing_room_does_not_blank_the_others(self) -> None:+ """The pause is the failing key's: one tenant or room that keeps+ failing must not turn every other room's events into `unknown`."""+ sink = _RecordingSink()+ service = _service(sink)+ messages = MessageTelemetry(+ telemetry=service,+ session_factory=None, # type: ignore[arg-type]+ )+ attempts: list[tuple[str, str]] = []++ async def _bad_room_fails(tenant_id: str, room_id: str) -> _RoomFacts:+ attempts.append((tenant_id, room_id))+ if room_id == "bad":+ raise ConnectionError("this room's lookup fails")+ return await _a_slack_room(tenant_id, room_id)++ messages._load_room_facts = _bad_room_fails # type: ignore[method-assign]+ other_tenant = str(uuid.uuid4())++ messages.observe(_said_in("bad"))+ messages.observe(_said_in("good"))+ messages.observe(+ ParticipantMessage(+ tenant_id=other_tenant,+ room_id="elsewhere",+ sender_client_id="someone",+ sender_role="human",+ has_attachment=False,+ in_thread=False,+ )+ )+ messages.observe(_said_in("bad"))+ reported = await _reported(messages, service, sink)++ assert [p["bridge_platform"] for _, p in reported] == [+ "unknown",+ "slack",+ "slack",+ "unknown",+ ]+ assert attempts == [+ (TENANT_ZERO, "bad"),+ (TENANT_ZERO, "good"),+ (other_tenant, "elsewhere"),+ ]++ async def test_failures_in_a_row_pause_every_lookup(+ self, caplog: pytest.LogCaptureFixture+ ) -> None:+ """Failing for several different keys with nothing succeeding between+ is the database, not a bad row: then every lookup waits."""+ sink = _RecordingSink()+ service = _service(sink)+ messages = MessageTelemetry(+ telemetry=service,+ session_factory=None, # type: ignore[arg-type]+ )+ attempts: list[str] = []++ async def _broken(tenant_id: str, room_id: str) -> None:+ attempts.append(room_id)+ raise ConnectionError("database unreachable")++ messages._load_room_facts = _broken # type: ignore[method-assign]++ with caplog.at_level("WARNING"):+ for room in ("a", "b", "c", "d", "e"):+ messages.observe(_said_in(room))+ reported = await _reported(messages, service, sink)++ assert attempts == ["a", "b", "c"]+ assert [p["bridge_platform"] for _, p in reported] == ["unknown"] * 5+ assert "failed 3 times in a row, so every lookup pauses" in caplog.text++ async def test_a_lookup_that_succeeds_resets_the_count(self) -> None:+ sink = _RecordingSink()+ service = _service(sink)+ messages = MessageTelemetry(+ telemetry=service,+ session_factory=None, # type: ignore[arg-type]+ )+ attempts: list[str] = []++ async def _only_good_answers(tenant_id: str, room_id: str) -> _RoomFacts:+ attempts.append(room_id)+ if room_id != "good":+ raise ConnectionError("this room's lookup fails")+ return await _a_slack_room(tenant_id, room_id)++ messages._load_room_facts = _only_good_answers # type: ignore[method-assign]++ for room in ("a", "b", "good", "c", "d"):+ messages.observe(_said_in(room))+ await _reported(messages, service, sink)++ assert attempts == ["a", "b", "good", "c", "d"]+++class TestCaching:+ async def test_a_senders_kind_is_read_once_while_cached(+ self, session_factory: async_sessionmaker[AsyncSession]+ ) -> None:+ world = await _world(session_factory)+ sink = _RecordingSink()+ service = _service(sink)+ messages = MessageTelemetry(telemetry=service, session_factory=session_factory)++ def _ask() -> None:+ messages.agent_addressed(+ tenant_id=TENANT_ZERO,+ room_id=world.room.id,+ sender_transport_user_id=world.humans[0].transport_user_id,+ from_platform=False,+ agent_metadata=None,+ agent_live=True,+ has_attachment=False,+ )++ _ask()+ await messages._queue.join()+ async with session_factory() as session:+ sender = await session.get(Client, world.humans[0].id)+ assert sender is not None+ sender.type = "agent"+ await session.commit()+ _ask()++ reported = await _reported(messages, service, sink)+ assert [p["sender_kind"] for _, p in reported] == ["user", "user"]++ async def test_an_agents_runtime_is_read_once_while_cached(+ self, session_factory: async_sessionmaker[AsyncSession]+ ) -> None:+ world = await _world(session_factory)+ sink = _RecordingSink()+ service = _service(sink)+ messages = MessageTelemetry(telemetry=service, session_factory=session_factory)++ messages.observe(_said(world, world.agent.client_id, "agent"))+ await messages._queue.join()+ async with session_factory() as session:+ agent = await session.get(Agent, world.agent.id)+ assert agent is not None+ agent.metadata_ = {"known_agent_type": "claude-code"}+ await session.commit()+ messages.observe(_said(world, world.agent.client_id, "agent"))++ reported = await _reported(messages, service, sink)+ assert [+ p["known_agent_type"]+ for name, p in reported+ if name == "switch_core.agent_message_sent"+ ] == ["codex", "codex"]++ async def test_a_room_that_cannot_be_found_is_looked_up_once(self) -> None:+ """A room deleted while its last messages were queued must not cost a+ query per message."""+ sink = _RecordingSink()+ service = _service(sink)+ messages = MessageTelemetry(+ telemetry=service,+ session_factory=None, # type: ignore[arg-type]+ )+ loads = 0++ async def _gone(tenant_id: str, room_id: str) -> None:+ nonlocal loads+ loads += 1+ return None++ messages._load_room_facts = _gone # type: ignore[method-assign]++ for _ in range(3):+ messages.observe(_said_in("deleted-room"))+ reported = await _reported(messages, service, sink)++ assert loads == 1+ assert [p["bridge_platform"] for _, p in reported] == ["unknown"] * 3++ async def test_a_sender_or_agent_that_cannot_be_found_is_looked_up_once(+ self, session_factory: async_sessionmaker[AsyncSession]+ ) -> None:+ """The miss is cached like a hit: a row that appears afterwards is not+ seen until the entry expires."""+ world = await _world(session_factory)+ sink = _RecordingSink()+ service = _service(sink)+ messages = MessageTelemetry(telemetry=service, session_factory=session_factory)+ late_sender = f"@late-{uuid.uuid4().hex[:8]}:test"+ async with session_factory() as session:+ unregistered = await _client(session, "agent")+ await session.commit()++ def _activity() -> None:+ messages.agent_addressed(+ tenant_id=TENANT_ZERO,+ room_id=world.room.id,+ sender_transport_user_id=late_sender,+ from_platform=False,+ agent_metadata=None,+ agent_live=True,+ has_attachment=False,+ )+ messages.observe(_said(world, unregistered.id, "agent"))++ _activity()+ await messages._queue.join()+ async with session_factory() as session:+ session.add(+ Client(transport_user_id=late_sender, display_name="late", type="user")+ )+ owner = await _agent(session, "codex")+ owner.client_id = unregistered.id+ await session.commit()+ _activity()++ reported = await _reported(messages, service, sink)+ assert [+ p["sender_kind"]+ for name, p in reported+ if name == "switch_core.agent_message_received"+ ] == ["unknown", "unknown"]+ assert [+ p["known_agent_type"]+ for name, p in reported+ if name == "switch_core.agent_message_sent"+ ] == ["unknown", "unknown"]+++class TestTtlCache:+ def test_an_entry_expires_after_its_age(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ clock = SimpleNamespace(now=100.0)+ monkeypatch.setattr(+ messages_module, "time", SimpleNamespace(monotonic=lambda: clock.now)+ )+ cache: _TtlCache[str, int] = _TtlCache(ttl_seconds=10, max_entries=5)+ cache.put("a", 1)++ clock.now += 9+ assert cache.get("a") == 1+ clock.now += 2+ assert cache.get("a") is None++ def test_the_least_recently_used_entry_goes_first(self) -> None:+ cache: _TtlCache[str, int] = _TtlCache(ttl_seconds=60, max_entries=2)+ cache.put("a", 1)+ cache.put("b", 2)+ cache.get("a")+ cache.put("c", 3)++ assert cache.get("b") is None+ assert cache.get("a") == 1+ assert cache.get("c") == 3+++class TestBackpressure:+ async def test_a_full_queue_drops_with_a_warning_and_a_final_count(+ self, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture+ ) -> None:+ """Bounded rather than growing inside the process it measures, and the+ loss is said out loud — including drops after the last warning."""+ monkeypatch.setattr(messages_module, "_QUEUE_SIZE", 1)+ sink = _RecordingSink()+ service = _service(sink)+ messages = MessageTelemetry(+ telemetry=service,+ session_factory=None, # type: ignore[arg-type]+ )+ messages._load_room_facts = _a_slack_room # type: ignore[method-assign]++ with caplog.at_level("WARNING"):+ for _ in range(4):+ messages.observe(_said_in("room"))+ reported = await _reported(messages, service, sink)++ behind = [+ r.getMessage()+ for r in caplog.records+ if "Message telemetry is behind" in r.getMessage()+ ]+ assert len(reported) == 1+ assert len(behind) == 2+ assert "1 event(s) dropped" in behind[0]+ assert "2 event(s) dropped" in behind[1]+++class TestFailingSenderAndRuntimeLookups:+ async def test_a_sender_lookup_that_fails_reports_unknown_and_pauses(+ self, caplog: pytest.LogCaptureFixture+ ) -> None:+ database = _Unreachable()+ sink = _RecordingSink()+ service = _service(sink)+ messages = MessageTelemetry(+ telemetry=service,+ session_factory=database, # type: ignore[arg-type]+ )+ messages._load_room_facts = _a_slack_room # type: ignore[method-assign]++ for sender in ("@alice:test", "@alice:test"):+ messages.agent_addressed(+ tenant_id=TENANT_ZERO,+ room_id="room",+ sender_transport_user_id=sender,+ from_platform=False,+ agent_metadata=None,+ agent_live=True,+ has_attachment=False,+ )+ with caplog.at_level("WARNING"):+ reported = await _reported(messages, service, sink)++ assert [p["sender_kind"] for _, p in reported] == ["unknown", "unknown"]+ assert database.attempts == 1+ assert "could not look up a message sender" in caplog.text++ async def test_a_runtime_lookup_that_fails_reports_unknown_and_pauses(+ self, caplog: pytest.LogCaptureFixture+ ) -> None:+ database = _Unreachable()+ sink = _RecordingSink()+ service = _service(sink)+ messages = MessageTelemetry(+ telemetry=service,+ session_factory=database, # type: ignore[arg-type]+ )+ messages._load_room_facts = _a_slack_room # type: ignore[method-assign]++ for _ in range(2):+ messages.observe(+ _said_in("room", sender_role="agent", sender_client_id="agent-client")+ )+ with caplog.at_level("WARNING"):+ reported = await _reported(messages, service, sink)++ assert [+ p["known_agent_type"]+ for name, p in reported+ if name == "switch_core.agent_message_sent"+ ] == ["unknown", "unknown"]+ assert database.attempts == 1+ assert "could not look up an agent's runtime" in caplog.text++ async def test_failures_past_the_pause_still_warn_once_a_minute(+ self, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture+ ) -> None:+ monkeypatch.setattr(messages_module, "_LOOKUP_BACKOFF_SECONDS", 0.0)+ sink = _RecordingSink()+ service = _service(sink)+ messages = MessageTelemetry(+ telemetry=service,+ session_factory=None, # type: ignore[arg-type]+ )+ attempts = 0++ async def _broken(tenant_id: str, room_id: str) -> None:+ nonlocal attempts+ attempts += 1+ raise ConnectionError("database unreachable")++ messages._load_room_facts = _broken # type: ignore[method-assign]++ with caplog.at_level("WARNING"):+ for _ in range(3):+ messages.observe(_said_in("room"))+ await _reported(messages, service, sink)++ assert attempts == 3+ assert (+ len([r for r in caplog.records if "could not look up" in r.getMessage()])+ == 1+ )++ async def test_rooms_that_cannot_be_found_warn_once(+ self, caplog: pytest.LogCaptureFixture+ ) -> None:+ sink = _RecordingSink()+ service = _service(sink)+ messages = MessageTelemetry(+ telemetry=service,+ session_factory=None, # type: ignore[arg-type]+ )++ async def _gone(tenant_id: str, room_id: str) -> None:+ return None++ messages._load_room_facts = _gone # type: ignore[method-assign]++ with caplog.at_level("WARNING"):+ messages.observe(_said_in("room-a"))+ messages.observe(_said_in("room-b"))+ reported = await _reported(messages, service, sink)++ assert [p["bridge_platform"] for _, p in reported] == ["unknown", "unknown"]+ assert (+ len([r for r in caplog.records if "found no room" in r.getMessage()]) == 1+ )+++class TestFailingReports:+ async def test_an_event_that_cannot_be_reported_logs_once_not_per_message(+ self,+ session_factory: async_sessionmaker[AsyncSession],+ caplog: pytest.LogCaptureFixture,+ ) -> None:+ """A bug that breaks every event would otherwise write a traceback per+ message."""+ world = await _world(session_factory)+ sink = _RecordingSink()+ service = _service(sink)+ messages = MessageTelemetry(telemetry=service, session_factory=session_factory)++ async def _broken(item: object) -> None:+ raise RuntimeError("bug in building the event")++ messages._report = _broken # type: ignore[method-assign]++ with caplog.at_level("WARNING"):+ for _ in range(3):+ messages.observe(_said(world, world.humans[0].id, "human"))+ await _reported(messages, service, sink)++ errors = [+ r+ for r in caplog.records+ if "Could not report a message event" in r.getMessage()+ ]+ assert len(errors) == 1+ assert errors[0].exc_info is not None+ # The two after it are counted at shutdown rather than lost.+ assert "2 more message event(s) could not be reported" in caplog.text++ async def test_a_sender_that_cannot_be_found_is_disclosed(+ self,+ session_factory: async_sessionmaker[AsyncSession],+ caplog: pytest.LogCaptureFixture,+ ) -> None:+ world = await _world(session_factory)+ sink = _RecordingSink()+ service = _service(sink)+ messages = MessageTelemetry(telemetry=service, session_factory=session_factory)++ with caplog.at_level("WARNING"):+ messages.agent_addressed(+ tenant_id=TENANT_ZERO,+ room_id=world.room.id,+ sender_transport_user_id="@nobody-we-know:test",+ from_platform=False,+ agent_metadata=None,+ agent_live=True,+ has_attachment=False,+ )+ [(_, properties)] = await _reported(messages, service, sink)++ assert properties["sender_kind"] == "unknown"+ assert "found no client for a message sender" in caplog.text+++class TestShutdown:+ async def test_events_after_shutdown_are_counted_not_silently_lost(+ self, caplog: pytest.LogCaptureFixture+ ) -> None:+ sink = _RecordingSink()+ service = _service(sink)+ messages = MessageTelemetry(+ telemetry=service,+ session_factory=None, # type: ignore[arg-type]+ )+ await messages.aclose()++ with caplog.at_level("WARNING"):+ for _ in range(3):+ messages.observe(+ ParticipantMessage(+ tenant_id=TENANT_ZERO,+ room_id="room",+ sender_client_id="someone",+ sender_role="human",+ has_attachment=False,+ in_thread=False,+ )+ )++ late = [+ r+ for r in caplog.records+ if "after message telemetry shut down" in r.getMessage()+ ]+ assert len(late) == 1++ async def test_events_dropped_during_the_drain_are_all_counted(+ self, caplog: pytest.LogCaptureFixture+ ) -> None:+ """Senders keep running while shutdown drains the queue. The warning is+ rate-limited, so the drops after its first line are tallied when the+ drain ends rather than never."""+ release = asyncio.Event()++ async def _slow_room(tenant_id: str, room_id: str) -> _RoomFacts:+ await release.wait()+ return await _a_slack_room(tenant_id, room_id)++ sink = _RecordingSink()+ service = _service(sink)+ messages = MessageTelemetry(+ telemetry=service,+ session_factory=None, # type: ignore[arg-type]+ )+ messages._load_room_facts = _slow_room # type: ignore[method-assign]+ messages.observe(_said_in("room"))+ with caplog.at_level("WARNING"):+ closing = asyncio.create_task(messages.aclose())+ await asyncio.sleep(0)+ for _ in range(3):+ messages.observe(_said_in("room"))+ release.set()+ await closing++ late = [+ r.getMessage()+ for r in caplog.records+ if "after message telemetry shut down" in r.getMessage()+ ]+ assert len(late) == 2+ assert "2 more message event(s)" in late[1]++ async def test_closing_stays_inside_the_callers_timeout(+ self, caplog: pytest.LogCaptureFixture+ ) -> None:+ """The worker is cancelled mid-lookup and its cleanup outlasts the+ shutdown budget. The caller's timeout must still fire: swallowing its+ cancellation would let shutdown run past the forced-exit grace."""+ sink = _RecordingSink()+ service = _service(sink)+ messages = MessageTelemetry(+ telemetry=service,+ session_factory=None, # type: ignore[arg-type]+ )+ looking_up = asyncio.Event()++ async def _slow_to_cancel(tenant_id: str, room_id: str) -> None:+ looking_up.set()+ try:+ await asyncio.sleep(10)+ except asyncio.CancelledError:+ await asyncio.sleep(1.0)+ raise++ messages._load_room_facts = _slow_to_cancel # type: ignore[method-assign]+ messages.observe(+ ParticipantMessage(+ tenant_id=TENANT_ZERO,+ room_id="room",+ sender_client_id="someone",+ sender_role="human",+ has_attachment=False,+ in_thread=False,+ )+ )+ await looking_up.wait()++ started = time.monotonic()+ with caplog.at_level("WARNING"), pytest.raises(TimeoutError):+ async with asyncio.timeout(0.7):+ await messages.aclose()+ elapsed = time.monotonic() - started++ assert elapsed < 0.9+ # The one the worker was cancelled in the middle of is not in the+ # queue any more, and is still lost.+ assert "1 message event(s) still queued at shutdown were dropped" in (+ caplog.text+ )+ assert messages._worker is not None+ await asyncio.wait({messages._worker})+++class TestOff:+ def test_enabled_follows_the_service(self) -> None:+ """What a caller with telemetry-only work checks before doing it."""+ for enabled in (True, False):+ messages = MessageTelemetry(+ telemetry=_service(_RecordingSink(), enabled=enabled),+ session_factory=None, # type: ignore[arg-type]+ )+ assert messages.enabled is enabled++ async def test_nothing_is_queued_or_looked_up_when_telemetry_is_off(+ self, session_factory: async_sessionmaker[AsyncSession]+ ) -> None:+ world = await _world(session_factory)+ sink = _RecordingSink()+ service = _service(sink, enabled=False)+ messages = MessageTelemetry(telemetry=service, session_factory=session_factory)++ messages.observe(_said(world, world.humans[0].id, "human"))+ messages.agent_addressed(+ tenant_id=TENANT_ZERO,+ room_id=world.room.id,+ sender_transport_user_id=world.humans[0].transport_user_id,+ from_platform=False,+ agent_metadata=None,+ agent_live=False,+ has_attachment=False,+ )++ assert messages._worker is None+ assert await _reported(messages, service, sink) == []diff --git a/core/tests/switch_core/telemetry/test_relay_attribute_cap.py b/core/tests/switch_core/telemetry/test_relay_attribute_cap.pyindex 6d9561f15..34ca456cd 100644--- a/core/tests/switch_core/telemetry/test_relay_attribute_cap.py+++ b/core/tests/switch_core/telemetry/test_relay_attribute_cap.py@@ -37,7 +37,7 @@ class _Capture:def __init__(self) -> None:self.sent: list[TelemetryRecord] = []- async def send(self, record: TelemetryRecord) -> None:+ def send(self, record: TelemetryRecord) -> None:self.sent.append(record)async def aclose(self) -> None:@@ -89,9 +89,14 @@ async def _attributes_on_the_wire(http = httpx.AsyncClient(transport=httpx.MockTransport(_handle))[record] = capture.sent- await OtlpRelaySink(client=OtlpClient("https://relay.example", 5, {}, http)).send(- record+ sink = OtlpRelaySink(+ client=OtlpClient("https://relay.example", 5, {}, http),+ flush_interval_seconds=60.0,+ max_batch=200,+ max_buffered=10_000,)+ sink.send(record)+ await sink.aclose()await http.aclose()return body["resourceLogs"][0]["scopeLogs"][0]["logRecords"][0]["attributes"]diff --git a/core/tests/switch_core/telemetry/test_service_and_sink.py b/core/tests/switch_core/telemetry/test_service_and_sink.pyindex 172845554..e98c91f14 100644--- a/core/tests/switch_core/telemetry/test_service_and_sink.py+++ b/core/tests/switch_core/telemetry/test_service_and_sink.py@@ -9,17 +9,35 @@ import httpximport pytestfrom switch_core.observability.otlp import OtlpClient+from switch_core.telemetry import sink as sink_modulefrom switch_core.telemetry.catalogue import TelemetryCatalogueErrorfrom switch_core.telemetry.service import TelemetryService, emit_safelyfrom switch_core.telemetry.sink import NullSink, OtlpRelaySink, TelemetryRecord+def _relay_sink(+ http: httpx.AsyncClient,+ *,+ flush_interval_seconds: float = 60.0,+ max_batch: int = 200,+ max_buffered: int = 10_000,+) -> OtlpRelaySink:+ """A relay sink whose timer will not fire inside a test unless asked to:+ a test posts by closing it, or by filling a batch."""+ return OtlpRelaySink(+ client=OtlpClient("https://relay.example", 5, {}, http),+ flush_interval_seconds=flush_interval_seconds,+ max_batch=max_batch,+ max_buffered=max_buffered,+ )++class _RecordingSink:def __init__(self) -> None:self.sent: list[TelemetryRecord] = []self.closed = False- async def send(self, record: TelemetryRecord) -> None:+ def send(self, record: TelemetryRecord) -> None:self.sent.append(record)async def aclose(self) -> None:@@ -156,7 +174,7 @@ class TestTagging:class TestFailuresDoNotReachTheCaller:async def test_a_sink_that_raises_does_not_surface(self) -> None:class _Broken:- async def send(self, record: TelemetryRecord) -> None:+ def send(self, record: TelemetryRecord) -> None:raise RuntimeError("relay on fire")async def aclose(self) -> None:@@ -197,8 +215,8 @@ class TestTheWireFormat:return handler(request) # type: ignore[operator]http = httpx.AsyncClient(transport=httpx.MockTransport(_handle))- sink = OtlpRelaySink(client=OtlpClient("https://relay.example", 5, {}, http))- await sink.send(+ sink = _relay_sink(http)+ sink.send(TelemetryRecord(name="switch_core.usage_snapshot",properties={"room_count": 7, "from_template": True, "kind": "user"},@@ -211,6 +229,7 @@ class TestTheWireFormat:timestamp_ns=1_700_000_000_000_000_000,))+ await sink.aclose()await http.aclose()return captured@@ -250,8 +269,8 @@ class TestTheWireFormat:return httpx.Response(200)http = httpx.AsyncClient(transport=httpx.MockTransport(_handle))- sink = OtlpRelaySink(client=OtlpClient("https://relay.example", 5, {}, http))- await sink.send(+ sink = _relay_sink(http)+ sink.send(TelemetryRecord(name="switch_core.connector_added",properties={@@ -267,6 +286,7 @@ class TestTheWireFormat:timestamp_ns=1_700_000_000_000_000_000,))+ await sink.aclose()await http.aclose()attributes = {a["key"]: a["value"] for a in self._record(body)["attributes"]}@@ -303,8 +323,9 @@ class TestTheWireFormat:return httpx.Response(200)http = httpx.AsyncClient(transport=httpx.MockTransport(_handle))- sink = OtlpRelaySink(client=OtlpClient("https://relay.example", 5, {}, http))- await sink.send(_a_record())+ sink = _relay_sink(http)+ sink.send(_a_record())+ await sink.aclose()await http.aclose()assert seen == ["https://relay.example/v1/logs"]@@ -318,8 +339,9 @@ class TestTheWireFormat:return httpx.Response(200)http = httpx.AsyncClient(transport=httpx.MockTransport(_handle))- sink = OtlpRelaySink(client=OtlpClient("https://relay.example", 5, {}, http))- await sink.send(_a_record())+ sink = _relay_sink(http)+ sink.send(_a_record())+ await sink.aclose()await http.aclose()assert "authorization" not in seen@@ -333,8 +355,9 @@ class TestTheRelayMisbehaving:http = httpx.AsyncClient(transport=httpx.MockTransport(handler) # type: ignore[arg-type])- sink = OtlpRelaySink(client=OtlpClient("https://relay.example", 5, {}, http))- await sink.send(_a_record())+ sink = _relay_sink(http)+ sink.send(_a_record())+ await sink.aclose()await http.aclose()async def test_a_refusal_is_swallowed(self) -> None:@@ -360,11 +383,41 @@ class TestTheRelayMisbehaving:with caplog.at_level("WARNING"):await self._send_against(lambda r: httpx.Response(- 200, json={"partialSuccess": {"rejectedLogRecords": "3"}}+ 200, json={"partialSuccess": {"rejectedLogRecords": "1"}}))- assert "was not sent" in caplog.text+ assert "rejected 1 of 1" in caplog.text++ async def test_a_partial_success_reports_what_was_lost_not_the_batch(+ self, caplog: pytest.LogCaptureFixture+ ) -> None:+ """One rejected record out of a batch is one lost, not the batch:+ reporting the whole batch as unsent misstates the loss and sends an+ operator after the wrong problem."""+ http = httpx.AsyncClient(+ transport=httpx.MockTransport(+ lambda r: httpx.Response(+ 200,+ json={+ "partialSuccess": {+ "rejectedLogRecords": "1",+ "errorMessage": "too many attributes",+ }+ },+ )+ )+ )+ sink = _relay_sink(http)+ for _ in range(3):+ sink.send(_a_record())+ with caplog.at_level("WARNING"):+ await sink.aclose()+ await http.aclose()++ assert "rejected 1 of 3" in caplog.text+ assert "too many attributes" in caplog.text+ assert "was not sent" not in caplog.textasync def test_an_empty_200_is_not_treated_as_a_rejection(self, caplog: pytest.LogCaptureFixture@@ -389,35 +442,404 @@ def _a_record() -> TelemetryRecord:)-class TestNullSink:- async def test_it_discards_and_closes(self) -> None:- sink = NullSink()- await sink.send(TelemetryRecord("switch_core.x", {}, {}, 1))+class TestBatching:+ """The message events fire once per message, so the relay sink posts many+ events per request rather than one request per event."""++ async def test_events_wait_and_go_together(self) -> None:+ bodies: list[dict] = []++ def _handle(request: httpx.Request) -> httpx.Response:+ bodies.append(json.loads(request.content))+ return httpx.Response(200)++ http = httpx.AsyncClient(transport=httpx.MockTransport(_handle))+ sink = _relay_sink(http)+ for name in ("switch_core.room_message_sent", "switch_core.agent_message_sent"):+ sink.send(TelemetryRecord(name, {}, _a_record().resource, 1))++ assert bodies == []+ await sink.aclose()+ await http.aclose()++ assert len(bodies) == 1+ records = bodies[0]["resourceLogs"][0]["scopeLogs"][0]["logRecords"]+ # Each record keeps its own name, in both places the relay reads it.+ assert [r["eventName"] for r in records] == [+ "switch_core.room_message_sent",+ "switch_core.agent_message_sent",+ ]+ assert [r["body"]["stringValue"] for r in records] == [+ r["eventName"] for r in records+ ]++ async def test_a_full_batch_goes_without_waiting_for_the_timer(self) -> None:+ posted = asyncio.Event()+ sizes: list[int] = []++ def _handle(request: httpx.Request) -> httpx.Response:+ body = json.loads(request.content)+ sizes.append(len(body["resourceLogs"][0]["scopeLogs"][0]["logRecords"]))+ posted.set()+ return httpx.Response(200)++ http = httpx.AsyncClient(transport=httpx.MockTransport(_handle))+ sink = _relay_sink(http, max_batch=3)+ for _ in range(3):+ sink.send(_a_record())++ async with asyncio.timeout(2):+ await posted.wait()await sink.aclose()+ await http.aclose()++ assert sizes == [3]++ async def test_events_arriving_mid_post_wait_for_the_next_flush(self) -> None:+ """A flush posts what it found and no more. Draining until empty would,+ under steady traffic, post back to back in batches of a few events and+ never return to waiting — the per-event request load batching is for."""+ sizes: list[int] = []+ in_flight = asyncio.Event()+ release = asyncio.Event()++ async def _handle(request: httpx.Request) -> httpx.Response:+ body = json.loads(request.content)+ sizes.append(len(body["resourceLogs"][0]["scopeLogs"][0]["logRecords"]))+ in_flight.set()+ await release.wait()+ return httpx.Response(200)++ http = httpx.AsyncClient(transport=httpx.MockTransport(_handle))+ sink = _relay_sink(http, max_batch=2)+ sink.send(_a_record())+ sink.send(_a_record())+ async with asyncio.timeout(2):+ await in_flight.wait()+ sink.send(_a_record())+ release.set()+ await asyncio.sleep(0.05)++ assert sizes == [2]+ await sink.aclose()+ await http.aclose()+ assert sizes == [2, 1]++ async def test_the_timer_posts_a_partial_batch(self) -> None:+ posted = asyncio.Event()++ def _handle(request: httpx.Request) -> httpx.Response:+ posted.set()+ return httpx.Response(200)++ http = httpx.AsyncClient(transport=httpx.MockTransport(_handle))+ sink = _relay_sink(http, flush_interval_seconds=0.01)+ sink.send(_a_record())++ async with asyncio.timeout(2):+ await posted.wait()+ await sink.aclose()+ await http.aclose()++ async def test_a_full_buffer_drops_rather_than_grows(+ self, caplog: pytest.LogCaptureFixture+ ) -> None:+ bodies: list[dict] = []++ def _handle(request: httpx.Request) -> httpx.Response:+ bodies.append(json.loads(request.content))+ return httpx.Response(200)++ http = httpx.AsyncClient(transport=httpx.MockTransport(_handle))+ sink = _relay_sink(http, max_batch=100, max_buffered=2)+ with caplog.at_level("WARNING"):+ for _ in range(3):+ sink.send(_a_record())+ await sink.aclose()+ await http.aclose()++ assert "buffer is full" in caplog.text+ sent = [+ r+ for body in bodies+ for r in body["resourceLogs"][0]["scopeLogs"][0]["logRecords"]+ ]+ assert len(sent) == 2++ async def test_drops_after_the_last_warning_are_reported_at_shutdown(+ self, caplog: pytest.LogCaptureFixture+ ) -> None:+ """The warning is rate-limited, so drops after it would otherwise never+ be logged if no later drop came along to carry them."""+ http = httpx.AsyncClient(+ transport=httpx.MockTransport(lambda r: httpx.Response(200))+ )+ sink = _relay_sink(http, max_batch=100, max_buffered=1)+ sink.send(_a_record())+ with caplog.at_level("WARNING"):+ for _ in range(4):+ sink.send(_a_record())+ await sink.aclose()+ await http.aclose()++ warnings = [+ r.getMessage() for r in caplog.records if "buffer is full" in r.getMessage()+ ]+ assert len(warnings) == 2+ assert "1 event(s) dropped" in warnings[0]+ assert "3 event(s) dropped" in warnings[1]++ async def test_a_batch_that_cannot_be_built_costs_only_itself(+ self, caplog: pytest.LogCaptureFixture+ ) -> None:+ """A flush takes the whole buffer at once, so one bad batch must not+ take the batches after it down with it."""+ posted: list[int] = []++ def _handle(request: httpx.Request) -> httpx.Response:+ body = json.loads(request.content)+ posted.append(len(body["resourceLogs"][0]["scopeLogs"][0]["logRecords"]))+ return httpx.Response(200)++ http = httpx.AsyncClient(transport=httpx.MockTransport(_handle))+ sink = _relay_sink(http, max_batch=1)+ post = sink._post+ calls = 0++ async def _first_one_breaks(batch: object) -> None:+ nonlocal calls+ calls += 1+ if calls == 1:+ raise ValueError("bug in building the batch")+ await post(batch) # type: ignore[arg-type]++ sink._post = _first_one_breaks # type: ignore[method-assign]+ for _ in range(3):+ sink.send(_a_record())+ with caplog.at_level("WARNING"):+ await sink.aclose()+ await http.aclose()++ assert posted == [1, 1]+ assert "batch of 1 event(s) could not be built" in caplog.text++ async def test_events_after_shutdown_are_warned_about_once(+ self, caplog: pytest.LogCaptureFixture+ ) -> None:+ http = httpx.AsyncClient(+ transport=httpx.MockTransport(lambda r: httpx.Response(200))+ )+ sink = _relay_sink(http)+ await sink.aclose()+ with caplog.at_level("WARNING"):+ for _ in range(3):+ sink.send(_a_record())+ await http.aclose()++ late = [+ r+ for r in caplog.records+ if "arrived after shutdown began" in r.getMessage()+ ]+ assert len(late) == 1++ async def test_events_dropped_during_the_final_flush_are_all_counted(+ self, caplog: pytest.LogCaptureFixture+ ) -> None:+ """Senders keep running while shutdown waits on the last POST. The+ warning is rate-limited, so the drops after its first line are tallied+ when the flush ends rather than never."""+ release = asyncio.Event()++ async def _slow(request: httpx.Request) -> httpx.Response:+ await release.wait()+ return httpx.Response(200)++ http = httpx.AsyncClient(transport=httpx.MockTransport(_slow))+ sink = _relay_sink(http)+ sink.send(_a_record())+ with caplog.at_level("WARNING"):+ closing = asyncio.create_task(sink.aclose())+ await asyncio.sleep(0)+ for _ in range(3):+ sink.send(_a_record())+ release.set()+ await closing+ await http.aclose()++ late = [+ r.getMessage()+ for r in caplog.records+ if "arrived after shutdown began" in r.getMessage()+ ]+ assert len(late) == 2+ assert "2 more telemetry event(s)" in late[1]++ async def test_a_failed_batch_names_what_it_lost(+ self, caplog: pytest.LogCaptureFixture+ ) -> None:+ http = httpx.AsyncClient(+ transport=httpx.MockTransport(lambda r: httpx.Response(503))+ )+ sink = _relay_sink(http)+ sink.send(_a_record())+ with caplog.at_level("WARNING"):+ await sink.aclose()+ await http.aclose()++ assert "switch_core.deployment_started" in caplog.textclass TestShutdown:- async def test_in_flight_sends_are_awaited(self) -> None:- """The events worth losing least are the ones emitted just before the- process goes away."""- started = asyncio.Event()+ """The last flush runs under a fraction of a second. What it cannot post is+ counted, and a slow request already in flight does not hold the rest."""++ async def test_a_slow_post_in_flight_does_not_hold_back_the_rest(+ self, caplog: pytest.LogCaptureFixture+ ) -> None:+ in_flight = asyncio.Event()+ posted: list[int] = []++ async def _handle(request: httpx.Request) -> httpx.Response:+ if not in_flight.is_set():+ in_flight.set()+ await asyncio.sleep(10)+ body = json.loads(request.content)+ posted.append(len(body["resourceLogs"][0]["scopeLogs"][0]["logRecords"]))+ return httpx.Response(200)- class _Slow:- def __init__(self) -> None:- self.finished = False+ http = httpx.AsyncClient(transport=httpx.MockTransport(_handle))+ sink = _relay_sink(http, max_batch=1)+ sink.send(_a_record())+ async with asyncio.timeout(2):+ await in_flight.wait()+ sink.send(_a_record())+ sink.send(_a_record())++ with caplog.at_level("WARNING"), pytest.raises(TimeoutError):+ async with asyncio.timeout(0.3):+ await sink.aclose()+ await http.aclose()- async def send(self, record: TelemetryRecord) -> None:- started.set()- await asyncio.sleep(0.05)- self.finished = True+ assert posted == [1, 1]+ assert (+ "1 telemetry event(s) were never sent: shutdown ran out of time"+ in caplog.text+ )- async def aclose(self) -> None:- return None+ async def test_what_the_timeout_cuts_off_is_counted(+ self, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture+ ) -> None:+ """Both the batches still waiting and the ones abandoned mid-request:+ a batch leaves the buffer before it is posted, so counting only what+ is left would report nothing lost for a request the timeout cut off."""+ monkeypatch.setattr(sink_module, "_SHUTDOWN_CONCURRENCY", 2)++ async def _hang(request: httpx.Request) -> httpx.Response:+ await asyncio.sleep(10)+ return httpx.Response(200)++ http = httpx.AsyncClient(transport=httpx.MockTransport(_hang))+ sink = _relay_sink(http, max_batch=2)+ for _ in range(5):+ sink.send(_a_record())++ with caplog.at_level("WARNING"), pytest.raises(TimeoutError):+ async with asyncio.timeout(0.2):+ await sink.aclose()+ await http.aclose()++ assert "5 telemetry event(s) were never sent" in caplog.text++ async def test_the_drop_tallies_survive_the_timeout(+ self, caplog: pytest.LogCaptureFixture+ ) -> None:+ async def _hang(request: httpx.Request) -> httpx.Response:+ await asyncio.sleep(10)+ return httpx.Response(200)++ http = httpx.AsyncClient(transport=httpx.MockTransport(_hang))+ sink = _relay_sink(http, max_batch=100, max_buffered=1)+ sink.send(_a_record())+ with caplog.at_level("WARNING"):+ for _ in range(3):+ sink.send(_a_record())+ with pytest.raises(TimeoutError):+ async with asyncio.timeout(0.2):+ await sink.aclose()+ await http.aclose()++ full = [+ r.getMessage() for r in caplog.records if "buffer is full" in r.getMessage()+ ]+ assert "2 event(s) dropped" in full[-1]++ async def test_a_relay_that_refuses_at_shutdown_is_not_fed_the_rest(+ self, caplog: pytest.LogCaptureFixture+ ) -> None:+ requests = 0++ def _refuse(request: httpx.Request) -> httpx.Response:+ nonlocal requests+ requests += 1+ return httpx.Response(503)++ http = httpx.AsyncClient(transport=httpx.MockTransport(_refuse))+ sink = _relay_sink(http, max_batch=1)+ for _ in range(10):+ sink.send(_a_record())+ with caplog.at_level("WARNING"):+ await sink.aclose()+ await http.aclose()++ assert requests <= sink_module._SHUTDOWN_CONCURRENCY+ assert (+ f"{10 - requests} telemetry event(s) were never sent: the relay "+ "refused a batch at shutdown"+ ) in caplog.text++ async def test_a_clean_shutdown_reports_nothing_lost(+ self, caplog: pytest.LogCaptureFixture+ ) -> None:+ http = httpx.AsyncClient(+ transport=httpx.MockTransport(lambda r: httpx.Response(200))+ )+ sink = _relay_sink(http, max_batch=1)+ for _ in range(10):+ sink.send(_a_record())+ with caplog.at_level("WARNING"):+ await sink.aclose()+ await http.aclose()++ assert "never sent" not in caplog.text+++class TestNullSink:+ async def test_it_discards_and_closes(self) -> None:+ sink = NullSink()+ sink.send(TelemetryRecord("switch_core.x", {}, {}, 1))+ await sink.aclose()- sink = _Slow()++class TestHandOff:+ async def test_the_sink_has_the_event_as_soon_as_emit_returns(self) -> None:+ """No task per event: the message events fire once per message, and the+ sink only buffers, so a task would be pure overhead on the loop."""+ sink = _RecordingSink()service = _service(sink)+service.emit("deployment_started", tenant_count=1)- await started.wait()++ assert [record.name for record in sink.sent] == [+ "switch_core.deployment_started"+ ]++ async def test_closing_the_service_closes_the_sink(self) -> None:+ """The sink posts what it still holds when closed: the events worth+ losing least are the ones emitted just before the process goes away."""+ sink = _RecordingSink()+ service = _service(sink)+await service.aclose()- assert sink.finished+ assert sink.closeddiff --git a/core/tests/switch_core/telemetry/test_snapshot.py b/core/tests/switch_core/telemetry/test_snapshot.pyindex 2391a3d04..79396eee1 100644--- a/core/tests/switch_core/telemetry/test_snapshot.py+++ b/core/tests/switch_core/telemetry/test_snapshot.py@@ -718,6 +718,145 @@ class TestMembership:assert counts.room_users_max == 3+class TestActivityTiers:+ """Three nested populations of chat identities: seen in a room at all,+ said anything, and said something in a room with an agent."""++ async def test_a_poster_whose_room_was_archived_is_still_seen(+ self, session_factory: async_sessionmaker[AsyncSession]+ ) -> None:+ """The tiers must nest, so nobody counts as having posted this week+ without also counting as seen in a room."""+ async with session_factory() as session:+ room = await _room(session, archived=True)+ agent = await _client(session, "agent")+ human = await _client(session, "user")+ await _join(session, agent, room)+ await _join(session, human, room)+ await _say(session, room, human, seq=1)++ counts = await _counts(session)++ assert counts.chat_identity_active_7d == 1+ assert counts.chat_identity_posted_7d == 1+ assert counts.chat_identity_in_room_count == 1++ async def test_a_poster_who_left_the_room_is_still_seen(+ self, session_factory: async_sessionmaker[AsyncSession]+ ) -> None:+ async with session_factory() as session:+ room = await _room(session)+ await _say(session, room, await _client(session, "user"), seq=1)++ counts = await _counts(session)++ assert counts.chat_identity_posted_7d == 1+ assert counts.chat_identity_in_room_count == 1++ async def test_a_member_who_posted_is_one_person_not_two(+ self, session_factory: async_sessionmaker[AsyncSession]+ ) -> None:+ async with session_factory() as session:+ room = await _room(session)+ human = await _client(session, "user")+ await _join(session, human, room)+ await _say(session, room, human, seq=1)++ counts = await _counts(session)++ assert counts.chat_identity_in_room_count == 1++ async def test_someone_who_left_and_went_quiet_is_no_longer_seen(+ self, session_factory: async_sessionmaker[AsyncSession]+ ) -> None:+ async with session_factory() as session:+ room = await _room(session)+ await _say(+ session,+ room,+ await _client(session, "user"),+ seq=1,+ when=NOW - timedelta(days=8),+ )++ counts = await _counts(session)++ assert counts.chat_identity_in_room_count == 0++ async def test_talking_only_to_people_is_posting_but_not_active(+ self, session_factory: async_sessionmaker[AsyncSession]+ ) -> None:+ async with session_factory() as session:+ room = await _room(session)+ speaker = await _client(session, "user")+ listener = await _client(session, "user")+ await _join(session, speaker, room)+ await _join(session, listener, room)+ await _say(session, room, speaker, seq=1)++ counts = await _counts(session)++ assert counts.chat_identity_in_room_count == 2+ assert counts.chat_identity_posted_1d == 1+ assert counts.chat_identity_active_1d == 0++ async def test_a_person_in_two_rooms_is_one_person(+ self, session_factory: async_sessionmaker[AsyncSession]+ ) -> None:+ async with session_factory() as session:+ human = await _client(session, "user")+ for _ in range(2):+ room = await _room(session)+ await _join(session, human, room)+ await _say(session, room, human, seq=1)++ counts = await _counts(session)++ assert counts.chat_identity_in_room_count == 1+ assert counts.chat_identity_posted_1d == 1++ async def test_only_people_in_live_rooms_count(+ self, session_factory: async_sessionmaker[AsyncSession]+ ) -> None:+ async with session_factory() as session:+ live = await _room(session)+ archived = await _room(session, archived=True)+ await _join(session, await _client(session, "user"), live)+ await _join(session, await _client(session, "user"), archived)+ await _join(session, await _client(session, "agent"), live)+ await _join(session, await _client(session, "bridge"), live)++ counts = await _counts(session)++ assert counts.chat_identity_in_room_count == 1++ async def test_posting_is_windowed(+ self, session_factory: async_sessionmaker[AsyncSession]+ ) -> None:+ async with session_factory() as session:+ room = await _room(session)+ human = await _client(session, "user")+ await _join(session, human, room)+ await _say(session, room, human, seq=1, when=NOW - timedelta(days=3))++ counts = await _counts(session)++ assert counts.chat_identity_posted_1d == 0+ assert counts.chat_identity_posted_7d == 1++ async def test_arriving_in_a_room_is_not_posting(+ self, session_factory: async_sessionmaker[AsyncSession]+ ) -> None:+ async with session_factory() as session:+ room = await _room(session)+ await _arrive(session, room, await _client(session, "user"), seq=1)++ counts = await _counts(session)++ assert counts.chat_identity_in_room_count == 1+ assert counts.chat_identity_posted_7d == 0++class TestNewlyActiveRooms:async def test_a_room_reports_when_a_person_first_speaks_in_it(self, session_factory: async_sessionmaker[AsyncSession]diff --git a/core/tests/switch_core/test_main_telemetry_drain.py b/core/tests/switch_core/test_main_telemetry_drain.pynew file mode 100644index 000000000..ac27ed44b--- /dev/null+++ b/core/tests/switch_core/test_main_telemetry_drain.py@@ -0,0 +1,91 @@+"""Telemetry at shutdown: bounded, and the sink always gets its turn."""++from __future__ import annotations++import asyncio+import time+from types import SimpleNamespace++import pytest++from switch_core import main+++class _Step:+ def __init__(self, *, hangs: bool, takes: float = 0.0) -> None:+ self.hangs = hangs+ self.takes = takes+ self.closed = False++ async def aclose(self) -> None:+ if self.hangs:+ await asyncio.sleep(10)+ await asyncio.sleep(self.takes)+ self.closed = True+++async def test_a_message_worker_that_overruns_does_not_cost_the_sink_its_flush() -> (+ None+):+ """The sink's final flush carries every kind of event, so the per-message+ worker using up a shared budget would lose far more than its own queue."""+ message_telemetry = _Step(hangs=True)+ telemetry = _Step(hangs=False)+ http = SimpleNamespace(closed=False)++ async def _close_http() -> None:+ http.closed = True++ http.aclose = _close_http++ started = time.monotonic()+ await main._drain_telemetry(telemetry, message_telemetry, http) # type: ignore[arg-type]+ elapsed = time.monotonic() - started++ assert telemetry.closed+ assert http.closed+ assert elapsed < main._TELEMETRY_DRAIN_SECONDS + 0.2+++async def test_the_sink_has_the_time_the_message_worker_did_not_use() -> None:+ """A fixed split would cut the sink off at its share however little of+ the second the worker took."""+ sink_takes = (+ main._TELEMETRY_DRAIN_SECONDS - main._MESSAGE_TELEMETRY_DRAIN_SECONDS + 0.2+ )+ telemetry = _Step(hangs=False, takes=sink_takes)++ await main._drain_telemetry(+ telemetry, # type: ignore[arg-type]+ _Step(hangs=False), # type: ignore[arg-type]+ None,+ )++ assert telemetry.closed+++async def test_the_whole_drain_stays_inside_its_budget() -> None:+ started = time.monotonic()+ await main._drain_telemetry(+ _Step(hangs=True), # type: ignore[arg-type]+ _Step(hangs=True), # type: ignore[arg-type]+ None,+ )++ assert time.monotonic() - started < main._TELEMETRY_DRAIN_SECONDS + 0.2+++async def test_a_step_that_fails_is_disclosed_and_the_sink_still_closes(+ caplog: pytest.LogCaptureFixture,+) -> None:+ class _Broken:+ async def aclose(self) -> None:+ raise RuntimeError("bug in closing")++ telemetry = _Step(hangs=False)++ with caplog.at_level("WARNING"):+ await main._drain_telemetry(telemetry, _Broken(), None) # type: ignore[arg-type]++ assert telemetry.closed+ assert "Reporting queued message events at shutdown failed" in caplog.textdiff --git a/core/tests/switch_core/transport/test_postgres_transport.py b/core/tests/switch_core/transport/test_postgres_transport.pyindex 394ee2464..9d0b81b0f 100644--- a/core/tests/switch_core/transport/test_postgres_transport.py+++ b/core/tests/switch_core/transport/test_postgres_transport.py@@ -40,6 +40,7 @@ from switch_core.db.stores.message_store import MessageStorefrom switch_core.db.stores.room_store import RoomStorefrom switch_core.db.stores.usage_store import UsageStorefrom switch_core.observability.metrics import MetricsRegistry, install, uninstall+from switch_core.observability.throttle import WarningThrottlefrom switch_core.provisioning.postgres import PostgresProvisioningfrom switch_core.tenant_context import current_tenant_id, tenant_scopefrom switch_core.transport import (@@ -51,8 +52,14 @@ from switch_core.transport import (TransportError,TransportHandlers,)+from switch_core.transport import postgres as postgres_modulefrom switch_core.transport.ephemeral import EphemeralBusfrom switch_core.transport.invites import InviteBus+from switch_core.transport.observer import (+ IgnoreParticipantMessages,+ ParticipantMessage,+ ParticipantMessageObserver,+)from switch_core.transport.postgres import DELIVERY_PAGE, PostgresTransportfrom switch_core.transport.room_cache import RoomCacheLimits, RoomDeliveryCachefrom tests.conftest import RLSHarness@@ -213,6 +220,7 @@ def _transport(ephemeral: EphemeralBus | None = None,invites: InviteBus | None = None,actor_role: str = "agent",+ message_observer: ParticipantMessageObserver | None = None,) -> PostgresTransport:"""A transport for `client_id`, acting in `tenant_id`.@@ -239,6 +247,7 @@ def _transport(invites=invites or InviteBus(),ephemeral=ephemeral or EphemeralBus(),room_cache=_room_cache(session_factory),+ message_observer=message_observer or IgnoreParticipantMessages(),)@@ -354,6 +363,193 @@ class TestMetering:assert await self._usage(session_factory) == [("messages", client_id, 1)]+class _Recorder:+ def __init__(self) -> None:+ self.seen: list[ParticipantMessage] = []++ def observe(self, message: ParticipantMessage) -> None:+ self.seen.append(message)+++class TestTheParticipantMessageObserver:+ """The observer hears what the tenant is metered for — what a participant+ chose to say — once per message, and only after it is committed."""++ async def test_a_metered_message_is_observed_with_what_the_transport_knows(+ self, session_factory: async_sessionmaker[AsyncSession]+ ) -> None:+ async with session_factory() as session:+ room_id, transport_room_id, client_id, user_id = await _make_room(session)+ await session.commit()+ recorder = _Recorder()+ transport = _transport(+ session_factory,+ client_id=client_id,+ user_id=user_id,+ actor_role="human",+ message_observer=recorder,+ )++ await transport.send_message(+ transport_room_id, "hello", sender_name="a", metered=True+ )++ assert recorder.seen == [+ ParticipantMessage(+ tenant_id=TENANT_ZERO_ID,+ room_id=room_id,+ sender_client_id=client_id,+ sender_role="human",+ has_attachment=False,+ in_thread=False,+ )+ ]++ async def test_a_reply_in_a_thread_says_so(+ self, session_factory: async_sessionmaker[AsyncSession]+ ) -> None:+ async with session_factory() as session:+ _, transport_room_id, client_id, user_id = await _make_room(session)+ await session.commit()+ recorder = _Recorder()+ transport = _transport(+ session_factory,+ client_id=client_id,+ user_id=user_id,+ message_observer=recorder,+ )++ root = await transport.send_message(+ transport_room_id, "root", sender_name="a", metered=True+ )+ await transport.send_message(+ transport_room_id,+ "reply",+ sender_name="a",+ metered=True,+ thread_root_id=root.event_id,+ )++ assert [m.in_thread for m in recorder.seen] == [False, True]++ async def test_what_switch_posts_on_someones_behalf_is_not_observed(+ self, session_factory: async_sessionmaker[AsyncSession]+ ) -> None:+ async with session_factory() as session:+ _, transport_room_id, client_id, user_id = await _make_room(session)+ await session.commit()+ recorder = _Recorder()+ transport = _transport(+ session_factory,+ client_id=client_id,+ user_id=user_id,+ message_observer=recorder,+ )++ await transport.send_message(+ transport_room_id, "unavailable", sender_name="a", metered=False+ )+ await transport.send_event(+ transport_room_id, "com.switch.report.tool_call", {"tool": "Bash"}+ )++ assert recorder.seen == []++ async def test_a_post_with_several_files_is_one_message(+ self, session_factory: async_sessionmaker[AsyncSession]+ ) -> None:+ async with session_factory() as session:+ _, transport_room_id, client_id, user_id = await _make_room(session)+ await session.commit()+ recorder = _Recorder()+ transport = _transport(+ session_factory,+ client_id=client_id,+ user_id=user_id,+ message_observer=recorder,+ )++ for index in range(3):+ await transport.send_media(+ transport_room_id,+ f"blob://{index}",+ f"file-{index}.txt",+ "text/plain",+ 10,+ sender_name="a",+ msgtype="m.file",+ metered=True,+ group={"id": "g1", "index": index, "total": 3},+ )++ assert len(recorder.seen) == 1+ assert recorder.seen[0].has_attachment is True++ async def test_an_observer_that_raises_does_not_unsend_the_message(+ self,+ session_factory: async_sessionmaker[AsyncSession],+ monkeypatch: pytest.MonkeyPatch,+ caplog: pytest.LogCaptureFixture,+ ) -> None:+ """The row is committed before the observer hears of it, so a failure+ there must not tell the sender its message was lost: the retry would+ post it twice. Caught, so it is counted, and logged once rather than+ once per message."""++ class _Broken:+ def observe(self, message: ParticipantMessage) -> None:+ raise RuntimeError("observer bug")++ registry = MetricsRegistry()+ install(registry)+ monkeypatch.setattr(+ postgres_module, "_observer_failures", WarningThrottle(60.0)+ )+ async with session_factory() as session:+ room_id, transport_room_id, client_id, user_id = await _make_room(session)+ await session.commit()+ transport = _transport(+ session_factory,+ client_id=client_id,+ user_id=user_id,+ message_observer=_Broken(),+ )++ try:+ with caplog.at_level("ERROR"):+ results = [+ await transport.send_message(+ transport_room_id, body, sender_name="a", metered=True+ )+ for body in ("still sent", "and this")+ ]+ failures = next(+ p+ for p in registry.collect()+ if p.name == "switch.messages.observer_failures"+ )+ finally:+ uninstall()++ assert all(result.event_id for result in results)+ async with session_factory() as session:+ stored = await session.scalars(+ select(Message).where(Message.room_id == room_id).order_by(Message.seq)+ )+ assert [m.body for m in stored] == ["still sent", "and this"]+ assert failures.numbers[0].value == 2.0+ assert (+ len(+ [+ r+ for r in caplog.records+ if "Message observer failed" in r.getMessage()+ ]+ )+ == 1+ )++class TestSending:async def test_a_message_becomes_a_row(self, session_factory: async_sessionmaker[AsyncSession]@@ -630,6 +826,7 @@ class TestMedia:invites=InviteBus(),ephemeral=EphemeralBus(),room_cache=_room_cache(session_factory),+ message_observer=IgnoreParticipantMessages(),)diff --git a/deploy/observability/dashboard.json b/deploy/observability/dashboard.jsonindex 6793ba358..a6e8245dd 100644--- a/deploy/observability/dashboard.json+++ b/deploy/observability/dashboard.json@@ -288,8 +288,8 @@{"definition": {"type": "timeseries",- "title": "Delivery failures per second",- "show_legend": false,+ "title": "Delivery and message-observer failures per second",+ "show_legend": true,"requests": [{"q": "sum:switch.messages.delivery_failures{$env,$service}.as_rate()",@@ -297,6 +297,13 @@"style": {"palette": "warm"}+ },+ {+ "q": "sum:switch.messages.observer_failures{$env,$service} by {actor}.as_rate()",+ "display_type": "bars",+ "style": {+ "palette": "orange"+ }}]},diff --git a/docs/old/observability.md b/docs/old/observability.mdindex 019328adc..4c019ad32 100644--- a/docs/old/observability.md+++ b/docs/old/observability.md@@ -236,6 +236,11 @@ handled in a way that leaves no trace.- **`switch.messages.send_failures`** — a send that raises before the commitwrites nothing, so the symptom is an absence, and an absence is what a quietroom looks like too.+- **`switch.messages.observer_failures`** — reporting a message for product+ telemetry raised, after the transport committed it or while an agent's+ consumer was delivering it. Caught, because the message itself is fine: a+ send that raised after its commit would be retried and posted twice. So the+ only trace is a rate-limited log line.- **`switch.agent.events_dropped`** — an agent's buffer discards events when itoverflows or when they age out. The agent is *told* it missed them, sonothing is hidden from the agent; nothing told anybody else.diff --git a/docs/old/telemetry-events.md b/docs/old/telemetry-events.mdindex cb83ccc5f..c360674b8 100644--- a/docs/old/telemetry-events.md+++ b/docs/old/telemetry-events.md@@ -225,7 +225,7 @@ server setting. The Console instead sends a bespoke `build` attributeserver equivalent. The two are deliberately different fields rather than onefield meaning different things on each side.-## Three kinds of event+## Four kinds of event**A daily snapshot**, one per deployment, carrying the counts that describe theinstallation. This answers "how much".@@ -237,28 +237,42 @@ seconds elapsed since install. This answers "how fast to value".failure: which platforms are in use, how often a bridge drops, whether oneconnector fails repeatedly before it works.+**Message events**, one per message, the only high-volume kind. These carry how+much is said and between whom — people to agents, agents back, people among+themselves — and the size of the room it was said in.+The snapshot exists because of the identifier rule. Without room ids,per-occurrence events can tell you *how much* happened but never *across howmany rooms* — counting distinct anything in Amplitude requires an identifier forthe thing being counted. Counting locally and reporting the total sidesteps it.-### Why there is no `message_sent` event--Message volume is reported as a count in the daily snapshot, not as an event per-message. Three reasons, in order of weight:--1. **Volume.** The relay path has no batching, no retry and no queue — one HTTP- request per event, fire-and-forget, because a desktop app emits a handful an- hour. A busy server emits thousands of messages an hour. Per-message events- would be a different class of load on a component not built for it.-2. **It answers nothing extra.** Without a room id, a per-message event supports- the same charts the snapshot count does.-3. **Content risk.** An event shaped around a message is the one most likely to- grow a property that reveals something about the message. Not having it- removes the temptation.--If per-message granularity is ever genuinely needed, it should arrive with-batching on the export path first.+### Message events, and what they cannot tell you++An earlier version of this note argued against an event per message, for three+reasons. Each is now answered rather than ignored:++1. **Volume.** The relay path used to post one request per event. The relay+ sink now batches: events wait up to five seconds, or until two hundred are+ queued, and go as one request. Its buffer is bounded, so a relay that is down+ costs events, never memory.+2. **The sender must not pay for it.** The transport and the agent consumer+ hand a small record to a queue and return. One worker looks up the room and+ the sender — cached, so a busy room is read once every five minutes rather+ than once per message — and emits. That queue is bounded too, and drops with+ a warning when it falls behind.+3. **Content risk.** The events carry the same kind of property as every+ other: closed sets, booleans and counts. There is no property a message body,+ a name or an id could reach, and the catalogue test enforces it.++**They count messages, not people.** No user id is ever sent, so in Amplitude+every event from a deployment belongs to one subject — the deployment — and+Amplitude's own "active users" counts deployments. How many *people* are active+comes only from the snapshot, which counts distinct people locally and reports+the total: `chat_identity_in_room_count` (seen in a room at all),+`chat_identity_posted_*` (said anything) and `chat_identity_active_*` (said+something in a room with an agent), with `user_active_*` counting the same+last tier as Switch accounts. The message events say how much those people+said, and to whom.## Definitions@@ -332,6 +346,9 @@ never invisible.| `chat_identity_count` | number | chat identities — Slack, Mattermost and other platform accounts — that exist, claimed by an account or not || `chat_identity_active_1d` | number | distinct chat identities that used a room with an agent in 24h. Most belong to no Switch account and one person may have several, which is why this runs above `user_active_1d`. Servers released before this change reported this figure as `user_active_1d` || `chat_identity_active_7d` | number | same over 7 days |+| `chat_identity_posted_1d` | number | distinct chat identities that said anything in any room in 24h, agent or no agent |+| `chat_identity_posted_7d` | number | same over 7 days |+| `chat_identity_in_room_count` | number | distinct chat identities Switch has seen in a room: a member of at least one unarchived room of any origin, or anyone who said anything in the last 7 days. Not a channel's whole audience — see "Tiers of activity" || `room_count` | number | **the headline figure** — unarchived rooms a *person* created || `room_agent_created_count` | number | unarchived rooms an agent created for itself || `room_system_created_count` | number | unarchived channels Switch adopted after being invited to them on a platform |@@ -384,6 +401,28 @@ participant *said*: chat messages, and commands a person typed on a platform.The automatic notice Switch posts under an agent's name when it cannot take arequest is excluded as well — the agent did not say it.+**Tiers of activity.** `chat_identity_in_room_count`,+`chat_identity_posted_7d` and `chat_identity_active_7d` nest: everyone Switch+has seen in a room, those who said anything, and those who said something in a+room with an agent. The nesting holds by construction — anyone who posted in+the last 7 days counts as seen, even if they have since left the room or it+was archived. `chat_identity_posted_7d - chat_identity_active_7d` is people+talking only to each other. All three count chat identities, so someone on both+Slack and Teams counts twice; only the top tier is also counted as accounts+(`user_active_*`). "In a room with an agent" stands in for "talked to an+agent": whether a message addressed one is decided per agent as it is+delivered, and is not stored where a count can read it.+`agent_message_received` is the per-message measure of that.++**What "seen in a room" cannot see.** Switch records a person in a room only+when they post, are added by name, or join the channel after Switch adopted+it. Nothing lists the members a channel already had, so someone who has only+ever read an adopted channel is in no count here, and+`chat_identity_in_room_count - chat_identity_posted_7d` is **not** the passive+audience: it is the people Switch happens to know about who stayed quiet,+usually a small fraction of the real one. Counting readers would take a+member sync on each platform, which no bridge does today.+**Turns rather than senders.** A turn is one message classified by who sent themessage *before* it in the same room. That is the only way to tell an agentanswering a person from two agents talking among themselves: a sender-only count@@ -622,6 +661,79 @@ migrations run in a different event loop from the server, so the answer wouldhave to be carried across on a module global, and it is operational triviarather than something the product wants to know.+### Message events — one per message++All three come from `telemetry/messages.py`, reported off the sender's path as+described [above](#message-events-and-what-they-cannot-tell-you). A post with+several files is one message. Room facts are cached for five minutes, so a+member count can be that far behind. A room, sender or agent whose lookup+fails is not looked up again for five seconds, and its events report it as+`unknown` meanwhile; every other room carries on. Three failures in a row,+with no lookup succeeding between them, pause every lookup for five seconds.+Either way a database incident shows in the charts as a burst of `unknown`+rather than as missing messages.++**`room_message_sent`** — everything a participant chose to say in a room: the+population the tenant is metered for, reported by the transport after the+write commits. What Switch posts on a participant's behalf — greetings, command+replies, offline and refusal notices, a template's kickoff — is not in it.+Neither is a command a person types (`!reset`, a slash command): commands are+not metered, although the snapshot's activity counts treat one as something+the person said.++| Property | Type |+|---|---|+| `sender_kind` | `user` \| `agent` — only people and agents are metered, so `platform` does not occur here |+| `bridge_platform` | platform, `none` for an internal room, `unknown` if the room could not be read |+| `channel_type` | channel type, or `unknown` if the room could not be read |+| `room_user_count` | number — human members of the room, `-1` if it could not be read |+| `room_agent_count` | number — agent members, `-1` if it could not be read |+| `has_attachment` | boolean |+| `in_thread` | boolean |++The average of `room_user_count` over these events is the room size a typical+message is said to — weighted by messages, not by rooms. **Filter out+`bridge_platform = unknown` before averaging**: those are rooms that could not+be read, and their `-1` would pull the mean down. Every property is required on+every event and the catalogue has no empty value, so `-1` is the marker. The+snapshot's `room_users_mean` is the per-room figure.++**`agent_message_received`** — a message an agent was asked to act on:+addressed to it (by name, alias, role, or a direct room) *and* let through its+addressing policy and budget, whether or not the agent was there to take it.+Reported by the agent's consumer, where that is decided, so a message naming+two agents is two of these and one `room_message_sent`. Switch's own+auto-replies are excluded; a request Switch carries on a person's behalf is+included, as `platform`. A message a hosted agent's mailbox already holds — the+same message seen again — is not counted twice, and nor is a multi-file post+whose files arrive too far apart to be delivered together: it is counted with+its first file. A typed command addressed to an agent is not counted.+`agent_live` splits being asked from being there: filter on it to count only+the requests an agent could act on as they arrived.++| Property | Type |+|---|---|+| `sender_kind` | `user` \| `agent` \| `platform` \| `unknown` |+| `known_agent_type` | the receiving agent's runtime |+| `bridge_platform` | platform |+| `channel_type` | channel type |+| `has_attachment` | boolean |+| `agent_live` | boolean — the agent had a live session for the room when the message arrived. False when it was offline, still starting, or its cloud worker refused the message (stopped, broken, or a full mailbox); always false for a `session_passive` agent, which reads its messages later rather than live |++**`agent_message_sent`** — a message an agent posted: its replies, status+updates, anything it chose to say. Every one is also a `room_message_sent` with+`sender_kind = agent`; this one carries the runtime as well. `room_user_count`+separates an agent answering people from agents talking among themselves.++| Property | Type |+|---|---|+| `known_agent_type` | the sending agent's runtime, or `unknown` if the agent could not be looked up |+| `bridge_platform` | platform |+| `channel_type` | channel type |+| `room_user_count` | number, `-1` if the room could not be read |+| `has_attachment` | boolean |+| `in_thread` | boolean |+### Closed value sets`bridge_platform`: `slack` | `mattermost` | `discord` | `teams` | `telegram` |@@ -630,6 +742,13 @@ rather than something the product wants to know.kept apart because collapsing them would misreport a Slack-bridged room asinternal-only whenever that lookup has a transient error.+`channel_type`: `channel_public` | `channel_private` | `direct` | `none` |+`unknown`, with `unknown` meaning what it does for `bridge_platform`. Only the+message events can carry it.++`sender_kind`: `user` | `agent` | `platform` | `unknown`. `platform` is Switch+itself speaking; `unknown` a sender whose client could not be looked up.+`outcome`: `success` | `failure`. `failure_reason` is an enumerated code perevent, `none` on success — never an exception message.@@ -729,6 +848,13 @@ the operational export follows. A value carrying the signal path is refused atstartup, because posting to `/v1/logs/v1/logs` would be a 404 the relay reportsonce per event into a log nobody is reading.+Product events are batched: they arrive at the sink up to five seconds after+they happen, several to a request. Whatever is buffered at shutdown is posted+on the way out, several batches at once, within a one-second shutdown budget:+the message worker's drain takes at most 0.4 seconds of it and the sink has+the rest. What cannot be posted in that time is lost, and the server log says+how many events that was.+The same sink serves the operational export, so one process shows both streams. Every event is printed as it arrives, decoded, with all of itsproperties. Create a room, register anagent, connect a bridge, send a message, and watch. The first snapshot lands@@ -756,7 +882,7 @@ that tests the vendor's own filtering.### Every event, and how to make it fire-Twenty-two events. Work down the list against a local sink+Work down the list against a local sink(`python scripts/otlp_sink.py`) and each one prints as it arrives with all ofits properties.@@ -823,6 +949,17 @@ Two things to set up first, or a third of the list cannot fire at all:(`bridges/agent/api/activity_routes.py`), with a UUID id. A repeat for thesame agent and session emits nothing, across restarts, for 7 days.+**Messages** — `telemetry/messages.py`, fed by `transport/postgres.py` and+`clients/agent_consumer.py`++- `room_message_sent` — any person or agent posting in any room. A post with+ three files is one event. A command reply or an offline notice is none.+- `agent_message_sent` — an agent posting, through its MCP tool or the HTTP+ API. Arrives beside a `room_message_sent` for the same message.+- `agent_message_received` — tag an agent, or write in a direct room with one.+ Tag two agents and there are two. Tag an agent whose policy refuses you and+ there are none.+**Connectors** — `bridges/collaboration/lifecycle_service.py`- `bridge_connected` — a bridge reaching its platform. Also fires withdiff --git a/scripts/otlp_sink.py b/scripts/otlp_sink.pyindex 1cf08a56d..f27e36fef 100644--- a/scripts/otlp_sink.py+++ b/scripts/otlp_sink.py@@ -106,9 +106,10 @@ def _summarise_logs(payload: dict[str, Any]) -> list[str]:for scope in resource.get("scopeLogs", []):records = scope.get("logRecords", [])# A product event is a log record carrying an event name, and it- # wants the opposite treatment from a log line: every property- # matters and there is one record per request, where a log batch- # is many records of which the first few are representative.+ # wants the opposite treatment from a log line: every record and+ # every property matters, even with many events batched into one+ # request, where a log batch is many records of which the first+ # few are representative.if records and "eventName" in records[0]:lines.extend(_summarise_events(records))continue