Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

Report per-message telemetry events and activity tiers (#719)

Christian McDermott212520c4d444
diff --git a/core/switch_core/clients/agent_consumer.py b/core/switch_core/clients/agent_consumer.py
index 54dee2e16..662a951a0 100644
--- a/core/switch_core/clients/agent_consumer.py
+++ b/core/switch_core/clients/agent_consumer.py
@@ -97,6 +97,9 @@ from switch_core.events import (
TaskUpdate,
)
from switch_core.gateway.known_agents import known_agent_for
+from switch_core.observability.catalogue import OBSERVER_FAILURES
+from switch_core.observability.metrics import metrics
+from switch_core.observability.throttle import WarningThrottle
from switch_core.transport import (
InboundMedia,
InboundMembership,
@@ -107,8 +110,14 @@ from switch_core.transport import (
if TYPE_CHECKING:
from sqlalchemy.ext.asyncio import AsyncSession
+ from switch_core.telemetry.messages import MessageTelemetry
+
logger = logging.getLogger(__name__)
+# One for the process rather than one per consumer: every agent has a
+# consumer, and a telemetry bug breaks the report for all of them.
+_report_failures = WarningThrottle(60.0)
+
# How long to hold an incomplete multi-attachment group before delivering the
# parts that did arrive, flagged as incomplete. Groups normally complete in
# milliseconds (one sender, back-to-back events); this is a safety net so a
@@ -185,6 +194,11 @@ class HostedNote:
refusal: str | None
deliver: bool
+ @property
+ def redelivered(self) -> bool:
+ """The mailbox already held this event: it was seen here before."""
+ return self.refusal is None and not self.deliver
+
_HOSTED_STOPPED_MESSAGE = (
"My cloud worker is stopped, so I did not process this message. "
@@ -390,6 +404,7 @@ class AgentConsumer(Consumer[AgentActor]):
hosted_launch_store: HostedLaunchStore,
connections: AgentConnectionRegistry,
frontend_base_url: str | None,
+ message_telemetry: MessageTelemetry,
) -> None:
super().__init__(actor=actor)
self._event_buffer = event_buffer
@@ -405,6 +420,7 @@ class AgentConsumer(Consumer[AgentActor]):
self._waking_notice_revisions: dict[str, int] = {}
self._unreachable_notice_revisions: dict[str, int] = {}
self._connections = connections
+ self._message_telemetry = message_telemetry
self._frontend_base_url = (
frontend_base_url.rstrip("/") if frontend_base_url else None
)
@@ -589,6 +605,7 @@ class AgentConsumer(Consumer[AgentActor]):
# that took a slot per question would multiply a single message into
# dozens of concurrent checkouts.
is_addressed = False
+ agent_live = False
refusal: str | None = None
unavailable: str | None = None
if self._addressed_without_lookup(event, meta) is not False:
@@ -599,14 +616,14 @@ class AgentConsumer(Consumer[AgentActor]):
gate = await self._gate_addressed(session, agent, event, meta)
is_addressed = gate.addressed
refusal = gate.refusal
- if (
- is_addressed
- and not self._triggered_by_auto_reply(event)
- and not await self._is_available(session, agent, meta.room_id)
- ):
- unavailable = await self._reply_when_unavailable_here(
- session, agent, meta, self._sender_handle(event)
+ if is_addressed and not self._triggered_by_auto_reply(event):
+ agent_live = await self._is_available(
+ session, agent, meta.room_id
)
+ if not agent_live:
+ unavailable = await self._reply_when_unavailable_here(
+ session, agent, meta, self._sender_handle(event)
+ )
text = event.body
@@ -680,6 +697,10 @@ class AgentConsumer(Consumer[AgentActor]):
):
self._unreachable_notice_revisions[meta.room_id] = launch.revision
unavailable = NOTICE_MESSAGES["unreachable"]
+ if is_addressed:
+ self._report_addressed(
+ meta, event, hosted, agent_live=agent_live, has_attachment=False
+ )
if refusal is not None:
await self._post_auto_reply(room.room_id, event, refusal, reply_thread_root)
@@ -883,10 +904,19 @@ class AgentConsumer(Consumer[AgentActor]):
) -> None:
reply_thread_root = thread_id if thread_id is not None else event.event_id
gated: Agent | None = None
+ agent_live = False
if is_addressed:
async with self.session_factory() as session:
agent = await self._fresh_agent(session)
gate = await self._gate_addressed(session, agent, event, meta)
+ # Only telemetry reads this here — the media path posts no
+ # unavailable notice — so it is not worth a query while off.
+ if (
+ gate.addressed
+ and self._message_telemetry.enabled
+ and not self._triggered_by_auto_reply(event)
+ ):
+ agent_live = await self._is_available(session, agent, meta.room_id)
if gate.addressed:
gated = agent
is_addressed = gate.addressed
@@ -894,7 +924,6 @@ class AgentConsumer(Consumer[AgentActor]):
await self._post_auto_reply(
room.room_id, event, gate.refusal, reply_thread_root
)
-
sender_kind: str | None = None
on_behalf_of: str | None = None
if PLATFORM_MARKER in event.content:
@@ -925,14 +954,19 @@ class AgentConsumer(Consumer[AgentActor]):
),
)
+ hosted: HostedNote | None = None
if gated is not None:
hosted = await self._note_hosted_addressed(gated, agent_event)
if hosted is not None and hosted.refusal is not None:
await self._post_auto_reply(
room.room_id, event, hosted.refusal, reply_thread_root
)
- if hosted is not None and not hosted.deliver:
- return
+ if is_addressed:
+ self._report_addressed(
+ meta, event, hosted, agent_live=agent_live, has_attachment=True
+ )
+ if hosted is not None and not hosted.deliver:
+ return
if logger.isEnabledFor(logging.DEBUG):
logger.debug(
@@ -1705,6 +1739,62 @@ class AgentConsumer(Consumer[AgentActor]):
def _triggered_by_auto_reply(event: InboundMessage) -> bool:
return bool(event.content.get(AUTO_REPLY_FLAG))
+ def _report_addressed(
+ self,
+ meta: RoomMeta,
+ event: InboundMessage,
+ hosted: HostedNote | None,
+ *,
+ agent_live: bool,
+ has_attachment: bool,
+ ) -> None:
+ """Count a message this agent was asked to act on.
+
+ Every message let through the addressing policy and budget, whether or
+ not the agent could take it, so an offline or stopped agent is still
+ asked. `agent_live` is the agent having a live session for the room,
+ and false as well when a hosted agent's mailbox refused the message:
+ a full mailbox or a lost provider turns it away even with a worker
+ attached. Not an auto-reply: Switch's notice that another agent is
+ offline or refused is not someone asking this one for something. Not
+ a redelivery the mailbox already holds: that request was counted when
+ it first arrived. Not the files of a multi-file post delivered without
+ its first part, which happens when a group times out incomplete and
+ the rest arrives later: the post is counted with its first part, as
+ the transport counts it, so it is one request however it is split.
+
+ Guarded like the transport's observer call, and counted the same way:
+ this runs on the delivery path, ahead of the enqueue, and a telemetry
+ bug must not cost the agent the message.
+ """
+ if self._triggered_by_auto_reply(event):
+ return
+ if hosted is not None and hosted.redelivered:
+ return
+ group = parse_attachment_group(event.content)
+ if group is not None and group[1] != 0:
+ return
+ try:
+ self._message_telemetry.agent_addressed(
+ tenant_id=self.tenant_id,
+ room_id=meta.room_id,
+ sender_transport_user_id=event.sender,
+ from_platform=PLATFORM_MARKER in event.content,
+ agent_metadata=self.agent.metadata_,
+ agent_live=agent_live and (hosted is None or hosted.refusal is None),
+ has_attachment=has_attachment,
+ )
+ except Exception:
+ metrics().increment(OBSERVER_FAILURES, {"actor": "agent"})
+ if (failures := _report_failures.note()) is not None:
+ logger.error(
+ "Could not report a message addressed to agent %s; it is "
+ "still delivered. %d failure(s) since the last error.",
+ self.agent.name,
+ failures,
+ exc_info=True,
+ )
+
async def _post_auto_reply(
self,
transport_room_id: str,
diff --git a/core/switch_core/clients/client_factory.py b/core/switch_core/clients/client_factory.py
index 8bc782f0f..05d13c035 100644
--- a/core/switch_core/clients/client_factory.py
+++ b/core/switch_core/clients/client_factory.py
@@ -18,6 +18,7 @@ from switch_core.messages.notify import MessageListener
from switch_core.transport import MessageTransport
from switch_core.transport.ephemeral import EphemeralBus
from switch_core.transport.invites import InviteBus
+from switch_core.transport.observer import ParticipantMessageObserver
from switch_core.transport.postgres import PostgresTransport
from switch_core.transport.room_cache import RoomDeliveryCache
@@ -39,6 +40,7 @@ class ClientFactory:
invites: InviteBus,
ephemeral: EphemeralBus,
room_cache: RoomDeliveryCache,
+ message_observer: ParticipantMessageObserver,
) -> None:
self._client_store = client_store
self._session_factory = session_factory
@@ -52,6 +54,7 @@ class ClientFactory:
self._ephemeral = ephemeral
# One for the process, so every member of a room shares it.
self._room_cache = room_cache
+ self._message_observer = message_observer
self._registry: dict[
str,
tuple[
@@ -127,4 +130,5 @@ class ClientFactory:
invites=self._invites,
ephemeral=self._ephemeral,
room_cache=self._room_cache,
+ message_observer=self._message_observer,
)
diff --git a/core/switch_core/main.py b/core/switch_core/main.py
index 1745f1773..f9c1e731d 100644
--- a/core/switch_core/main.py
+++ b/core/switch_core/main.py
@@ -6,7 +6,7 @@ import logging
import os
import signal
import time
-from collections.abc import AsyncIterator
+from collections.abc import AsyncIterator, Coroutine
from contextlib import asynccontextmanager
from functools import partial
from pathlib import Path
@@ -169,6 +169,7 @@ from switch_core.session_activity.maintenance import (
)
from switch_core.session_activity.outcomes import ApprovalOutcomes
from switch_core.session_activity.service import AgentSessionActivityService
+from switch_core.telemetry.messages import MessageTelemetry
from switch_core.telemetry.reporter import SnapshotReporter
from switch_core.telemetry.service import TelemetryService
from switch_core.telemetry.setup import build_telemetry
@@ -288,29 +289,34 @@ async def _stop_management(management: Management) -> None:
# The innermost of three nested budgets: under
# `observability.logs.SHUTDOWN_FLUSH_SECONDS`, itself under
-# `_FORCED_EXIT_GRACE_SECONDS`.
+# `_FORCED_EXIT_GRACE_SECONDS`. This and the log flush already fill that
+# window, so neither can grow. The per-message worker gets at most
+# `_MESSAGE_TELEMETRY_DRAIN_SECONDS` and the sink the rest, so a slow worker
+# can never cost the sink its final flush, which carries every kind of event,
+# and whatever the worker does not use is the sink's.
_TELEMETRY_DRAIN_SECONDS = 1.0
+_MESSAGE_TELEMETRY_DRAIN_SECONDS = 0.4
async def _drain_telemetry(
- telemetry: TelemetryService, http_client: httpx.AsyncClient | None
+ telemetry: TelemetryService,
+ message_telemetry: MessageTelemetry,
+ http_client: httpx.AsyncClient | None,
) -> None:
"""Let in-flight product events finish, then close their client.
Never raises and never overruns: a relay that stopped answering must not
hold the process past the point where it is killed.
"""
- try:
- async with asyncio.timeout(_TELEMETRY_DRAIN_SECONDS):
- await telemetry.aclose()
- except TimeoutError:
- logger.warning(
- "Gave up waiting for in-flight telemetry after %.1fs; those events "
- "are lost.",
- _TELEMETRY_DRAIN_SECONDS,
- )
- except Exception:
- logger.warning("Telemetry shutdown failed; continuing.", exc_info=True)
+ loop = asyncio.get_running_loop()
+ deadline = loop.time() + _TELEMETRY_DRAIN_SECONDS
+ # First, because what it still holds is emitted through the sink.
+ await _close_by(
+ message_telemetry.aclose(),
+ loop.time() + _MESSAGE_TELEMETRY_DRAIN_SECONDS,
+ "queued message events",
+ )
+ await _close_by(telemetry.aclose(), deadline, "buffered telemetry events")
if http_client is not None:
try:
await http_client.aclose()
@@ -318,6 +324,22 @@ async def _drain_telemetry(
logger.warning("Telemetry HTTP client did not close.", exc_info=True)
+async def _close_by(
+ closing: Coroutine[object, object, None], deadline: float, what: str
+) -> None:
+ """Await one telemetry shutdown step until `deadline`, on the loop's clock.
+ Never raises. The step logs how much it lost itself."""
+ try:
+ async with asyncio.timeout_at(deadline):
+ await closing
+ except TimeoutError:
+ logger.warning("Gave up waiting for %s at shutdown.", what)
+ except Exception:
+ logger.warning(
+ "Reporting %s at shutdown failed; continuing.", what, exc_info=True
+ )
+
+
async def _snapshot_loop(reporter: SnapshotReporter) -> None:
# `no_tenant` for the reason every other long-lived task does it: the
# snapshot binds each tenant in turn as it counts, and must not inherit
@@ -519,6 +541,9 @@ async def run(config: SwitchConfig) -> None:
telemetry, installed_at, telemetry_http = await build_telemetry(
config, session_factory, switch_core_version()
)
+ message_telemetry = MessageTelemetry(
+ telemetry=telemetry, session_factory=session_factory
+ )
# ── Switch Trust guardrails ──────────────────────────────────────────────
trust_client, trust_http = build_trust_client(config)
@@ -594,6 +619,7 @@ async def run(config: SwitchConfig) -> None:
invites=invites,
ephemeral=ephemeral,
room_cache=room_cache,
+ message_observer=message_telemetry,
)
client_factory.register(
"agent",
@@ -611,6 +637,7 @@ async def run(config: SwitchConfig) -> None:
hosted_launch_store=HostedLaunchStore(),
connections=connections,
frontend_base_url=config.frontend_base_url,
+ message_telemetry=message_telemetry,
)
# Members that only write: a person on another platform, and a bridge's own
# identity (its reader, the WorkspaceConsumer, is built by the bridge).
@@ -941,7 +968,7 @@ async def run(config: SwitchConfig) -> None:
# teardown runs inside `_FORCED_EXIT_GRACE_SECONDS` and a
# product event is the least valuable thing in it.
await protocol.sessions.aclose()
- await _drain_telemetry(telemetry, telemetry_http)
+ await _drain_telemetry(telemetry, message_telemetry, telemetry_http)
if trust_http is not None:
await trust_http.aclose()
await observability.aclose()
diff --git a/core/switch_core/observability/catalogue.py b/core/switch_core/observability/catalogue.py
index 6230b0ffe..85c19df77 100644
--- a/core/switch_core/observability/catalogue.py
+++ b/core/switch_core/observability/catalogue.py
@@ -231,6 +231,17 @@ DELIVERY_FAILURES = _spec(
"loop it was.",
"actor",
)
+OBSERVER_FAILURES = _spec(
+ "switch.messages.observer_failures",
+ "sum",
+ "{failure}",
+ "Reports about a message that raised: the transport telling its observer "
+ "after the commit, or an agent's consumer reporting a message addressed to "
+ "it. The message is unaffected and only the report is lost, so these are "
+ "caught by design and invisible without a counter. `actor` is who was "
+ "writing or reading.",
+ "actor",
+)
DELIVERY_LAG = _spec(
"switch.messages.delivery_lag",
"histogram",
@@ -550,6 +561,7 @@ CATALOGUE: dict[str, MetricSpec] = {
MESSAGES_DELIVERED,
SEND_FAILURES,
DELIVERY_FAILURES,
+ OBSERVER_FAILURES,
DELIVERY_LAG,
DELIVERY_CACHE_READS,
DELIVERY_CACHE_FILLS,
diff --git a/core/switch_core/observability/otlp.py b/core/switch_core/observability/otlp.py
index 4158487e8..831608d55 100644
--- a/core/switch_core/observability/otlp.py
+++ b/core/switch_core/observability/otlp.py
@@ -100,6 +100,20 @@ class OtlpSendError(RuntimeError):
"""A payload did not reach the collector."""
+class OtlpPartialRejection(OtlpSendError):
+ """The collector took the payload but refused `rejected` of its records.
+
+ A subclass, so a caller that only cares whether everything arrived can
+ keep catching :class:`OtlpSendError`; one that batches can tell how much
+ of the batch was lost.
+ """
+
+ def __init__(self, url: str, rejected: int, reason: str) -> None:
+ super().__init__(f"POST {url} rejected {rejected} record(s): {reason}")
+ self.rejected = rejected
+ self.reason = reason
+
+
def _otlp_value(value: AttributeValue) -> dict[str, Any]:
# `bool` first: it subclasses `int`, and as 1/0 a yes/no becomes something
# a receiver will average.
@@ -414,7 +428,6 @@ def _raise_on_partial_rejection(url: str, response: httpx.Response) -> None:
if not rejected or int(rejected) == 0:
return
- raise OtlpSendError(
- f"POST {url} rejected {rejected} record(s): "
- f"{partial.get('errorMessage') or 'no reason given'}"
+ raise OtlpPartialRejection(
+ url, int(rejected), partial.get("errorMessage") or "no reason given"
)
diff --git a/core/switch_core/observability/throttle.py b/core/switch_core/observability/throttle.py
new file mode 100644
index 000000000..995520f3e
--- /dev/null
+++ b/core/switch_core/observability/throttle.py
@@ -0,0 +1,42 @@
+"""A condition that can hold for every message, logged at most once a minute.
+
+Code on the message path runs once per message, so a condition that persists
+— a full buffer, a database that will not answer, an observer that raises —
+would otherwise write one warning per message and bury the server's own log
+at the moment an operator needs it most.
+"""
+
+from __future__ import annotations
+
+import time
+
+
+class WarningThrottle:
+ """Counts occurrences of one condition and says when to log them.
+
+ `note` records an occurrence. It returns how many there have been since
+ the last time it said to log, this one included, when the interval has
+ passed, and None while it has not. The first occurrence always logs.
+
+ `take_pending` hands back what was counted but not yet logged, for a
+ caller about to stop that would otherwise lose the tally of a quiet
+ stretch at the end.
+ """
+
+ def __init__(self, interval_seconds: float) -> None:
+ self._interval = interval_seconds
+ self._pending = 0
+ self._last_logged: float | None = None
+
+ def note(self) -> int | None:
+ self._pending += 1
+ now = time.monotonic()
+ if self._last_logged is not None and now - self._last_logged < self._interval:
+ return None
+ count, self._pending = self._pending, 0
+ self._last_logged = now
+ return count
+
+ def take_pending(self) -> int:
+ count, self._pending = self._pending, 0
+ return count
diff --git a/core/switch_core/telemetry/catalogue.py b/core/switch_core/telemetry/catalogue.py
index 82a150434..900bff29f 100644
--- a/core/switch_core/telemetry/catalogue.py
+++ b/core/switch_core/telemetry/catalogue.py
@@ -90,7 +90,9 @@ BRIDGE_PLATFORM = one_of(
"slack", "mattermost", "discord", "teams", "telegram", "none", "unknown"
)
-CHANNEL_TYPE = one_of("channel_public", "channel_private", "direct", "none")
+# `unknown` as for BRIDGE_PLATFORM: the room exists and the lookup that would
+# have named its type failed, which is not the same as it having none.
+CHANNEL_TYPE = one_of("channel_public", "channel_private", "direct", "none", "unknown")
# All four connection models in `bridges/agent/protocol/types.py`. Missing one
# drops the registration event for every agent that uses it.
@@ -103,8 +105,18 @@ AGENT_TYPE = one_of(
# agent registered without declaring one at all.
KNOWN_AGENT_TYPE = one_of("claude-code", "codex", "opencode", "other", "none")
+# The runtime as `agent_message_sent` reports it. That event looks the sending
+# agent up after the fact, so it has a third answer: `unknown`, for a lookup
+# that failed or found no agent.
+LOOKED_UP_AGENT_TYPE = one_of(*KNOWN_AGENT_TYPE.values, "unknown")
+
ACTOR_KIND = one_of("user", "agent", "system")
+# Who said a message. `platform` is Switch itself speaking — a template's
+# kickoff, or a request it carries on a person's behalf — and `unknown` a
+# sender whose client could not be looked up.
+SENDER_KIND = one_of("user", "agent", "platform", "unknown")
+
OUTCOME = one_of("success", "failure")
# The four built-in types. A user-defined type's slug is free text, so it
@@ -196,6 +208,17 @@ _SNAPSHOT_COUNTS = (
"turn_human_to_agent_1d",
"turn_agent_to_human_1d",
"turn_agent_to_agent_1d",
+ # The tiers of activity, as chat identities. `chat_identity_active_*`
+ # above is the most active (spoke in a room with an agent in it);
+ # `chat_identity_posted_*` is anyone who said anything in any room;
+ # `chat_identity_in_room_count` is everyone Switch has seen in a room: a
+ # member of a live room, or anyone who posted in the last seven days. Not
+ # the channels' full audience — Switch records a person only once they
+ # post, are added by name, or join after the channel was adopted, so a
+ # member who has only ever read is not in it.
+ "chat_identity_posted_1d",
+ "chat_identity_posted_7d",
+ "chat_identity_in_room_count",
"attachment_count_1d",
# Stock rather than flow. The `*_attached_count` figures sit beside the
# totals because the gap between made and used is the signal.
@@ -360,6 +383,56 @@ CATALOGUE: Mapping[str, Mapping[str, PropertyType]] = {
# in the failure code and nothing alike in the time.
"duration_ms": NUMBER,
},
+ # ── Messages ─────────────────────────────────────────────────────────────
+ # One per message, unlike everything above. They carry no identifier, so
+ # they answer "how much, and between whom", never "who": distinct people
+ # come from the snapshot's `user_*` and `chat_identity_*` counts. A multi-file post is one
+ # message, not one per file.
+ #
+ # Everything a participant chose to say in a room — the population the
+ # tenant is metered for. Notices Switch posts on someone's behalf are not
+ # in it. The room sizes are its members right now, so averaging them over
+ # these events gives the room size a typical message is said to — once
+ # `bridge_platform = unknown` is filtered out, since a room that could not
+ # be read reports its sizes as -1.
+ "room_message_sent": {
+ "sender_kind": SENDER_KIND,
+ "bridge_platform": BRIDGE_PLATFORM,
+ "channel_type": CHANNEL_TYPE,
+ "room_user_count": NUMBER,
+ "room_agent_count": NUMBER,
+ "has_attachment": BOOLEAN,
+ "in_thread": BOOLEAN,
+ },
+ # A message an agent was asked to act on: addressed to it and let through
+ # its addressing policy and budget, whether or not the agent was there to
+ # take it. One per agent addressed, so a message naming two agents is two
+ # of these and one `room_message_sent`; a message a hosted agent's mailbox
+ # already holds is not counted again, nor a multi-file post delivered in
+ # two pieces. `agent_live` is whether the agent had a live session for the
+ # room when it arrived: false for one that was offline, still starting, or
+ # whose cloud worker refused the message, and always for a session_passive
+ # agent, which reads its messages later rather than live.
+ "agent_message_received": {
+ "sender_kind": SENDER_KIND,
+ "known_agent_type": KNOWN_AGENT_TYPE,
+ "bridge_platform": BRIDGE_PLATFORM,
+ "channel_type": CHANNEL_TYPE,
+ "has_attachment": BOOLEAN,
+ "agent_live": BOOLEAN,
+ },
+ # A message an agent posted: its replies, and anything else it chose to
+ # say. Also a `room_message_sent` with `sender_kind = agent`; a separate
+ # event so it can carry the runtime. `room_user_count` separates an agent
+ # answering people from agents talking among themselves.
+ "agent_message_sent": {
+ "known_agent_type": LOOKED_UP_AGENT_TYPE,
+ "bridge_platform": BRIDGE_PLATFORM,
+ "channel_type": CHANNEL_TYPE,
+ "room_user_count": NUMBER,
+ "has_attachment": BOOLEAN,
+ "in_thread": BOOLEAN,
+ },
# ── Resources, keys and groups ───────────────────────────────────────────
# Creating is intent, attaching is use, and the gap between them is the
# signal — hence both.
diff --git a/core/switch_core/telemetry/messages.py b/core/switch_core/telemetry/messages.py
new file mode 100644
index 000000000..a68497903
--- /dev/null
+++ b/core/switch_core/telemetry/messages.py
@@ -0,0 +1,562 @@
+"""The per-message events: what is said in rooms, and to and by agents.
+
+Everything else in the catalogue fires a handful of times a day. These fire
+once per message, so they are the only events where reporting could slow the
+thing being reported on. Three rules keep it off that path:
+
+- **The sender never waits.** The transport and the agent consumer hand over a
+ small record and return. One worker task looks up what the event needs — the
+ room's platform, type and membership, who the sender is — and emits.
+- **Lookups are cached.** A busy room says a lot and changes membership
+ rarely, so a room is read at most once per `_CACHE_TTL_SECONDS` rather than
+ once per message. The member counts can therefore be that far behind. A
+ lookup that finds nothing is cached too, as `unknown`: a room deleted while
+ its last messages were queued must not cost a query per message.
+- **A failing lookup is left alone.** A room, sender or agent whose lookup
+ failed is not looked up again for `_LOOKUP_BACKOFF_SECONDS`, and events
+ report it as `unknown` meanwhile. Only that key: one tenant or room that
+ keeps failing must not blank every other. When lookups fail
+ `_FAILURES_TO_PAUSE_ALL` times in a row, with none succeeding between, the
+ database itself is in trouble and every lookup pauses for that long. Either
+ way the failure is logged at most once a minute, rather than the worker
+ retrying a struggling database, and logging a traceback, once per message.
+
+The queue is bounded: a worker that falls behind drops events with a warning
+rather than growing a backlog inside the process it is measuring.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import contextvars
+import logging
+import time
+from collections import OrderedDict
+from dataclasses import dataclass
+
+from sqlalchemy import and_, func, select
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
+
+from switch_core.db.models import Agent, Client, ClientRoom, CollaborationBridge, Room
+from switch_core.db.session_scope import tenant_session
+from switch_core.observability.throttle import WarningThrottle
+from switch_core.telemetry.service import TelemetryService
+from switch_core.telemetry.snapshot import (
+ AGENT_CLIENT_TYPE,
+ HUMAN_CLIENT_TYPE,
+ normalise_channel_type,
+ normalise_known_agent_type,
+ normalise_platform,
+)
+from switch_core.transport.observer import ParticipantMessage
+
+logger = logging.getLogger(__name__)
+
+_QUEUE_SIZE = 10_000
+_CACHE_TTL_SECONDS = 300.0
+_CACHE_MAX_ENTRIES = 10_000
+_WARNING_INTERVAL_SECONDS = 60.0
+_LOOKUP_BACKOFF_SECONDS = 5.0
+# Failures in a row, from any keys, that read as the database failing rather
+# than one bad row or tenant. A success between them resets the count.
+_FAILURES_TO_PAUSE_ALL = 3
+# Inside `main._MESSAGE_TELEMETRY_DRAIN_SECONDS`, leaving the rest of it for
+# cancelling the worker.
+_DRAIN_SECONDS = 0.3
+
+# The transport's `ActorRole` as the catalogue's `sender_kind`. A system or
+# bridge writer is Switch itself.
+_ROLE_KIND = {
+ "human": "user",
+ "agent": "agent",
+ "system": "platform",
+ "bridge": "platform",
+}
+
+# `clients.type` as `sender_kind`. `admin` is Switch's own client.
+_CLIENT_TYPE_KIND = {
+ HUMAN_CLIENT_TYPE: "user",
+ AGENT_CLIENT_TYPE: "agent",
+ "admin": "platform",
+ "bridge": "platform",
+}
+
+# An agent whose runtime could not be looked up. Not `none`, which is a real
+# answer: an agent that declares no runtime.
+_UNKNOWN_RUNTIME = "unknown"
+
+
+@dataclass(frozen=True)
+class _RoomFacts:
+ bridge_platform: str
+ channel_type: str
+ user_count: int
+ agent_count: int
+
+
+# A room whose lookup failed. Reported rather than dropped, so a lookup
+# problem shows as `unknown` in the charts instead of as fewer messages; -1
+# rather than 0 so it cannot be read as an empty room. The catalogue has no
+# empty value, so an average of the counts has to filter on
+# `bridge_platform != unknown` — the docs say so.
+_UNKNOWN_ROOM = _RoomFacts(
+ bridge_platform="unknown", channel_type="unknown", user_count=-1, agent_count=-1
+)
+
+
+@dataclass(frozen=True)
+class _AgentAddressed:
+ tenant_id: str
+ room_id: str
+ sender_transport_user_id: str
+ from_platform: bool
+ known_agent_type: str
+ agent_live: bool
+ has_attachment: bool
+
+
+class _TtlCache[K, V]:
+ """Least-recently-used, with entries expiring after a fixed age."""
+
+ def __init__(self, ttl_seconds: float, max_entries: int) -> None:
+ self._ttl = ttl_seconds
+ self._max = max_entries
+ self._entries: OrderedDict[K, tuple[float, V]] = OrderedDict()
+
+ def get(self, key: K) -> V | None:
+ entry = self._entries.get(key)
+ if entry is None:
+ return None
+ expires_at, value = entry
+ if expires_at < time.monotonic():
+ del self._entries[key]
+ return None
+ self._entries.move_to_end(key)
+ return value
+
+ def put(self, key: K, value: V) -> None:
+ self._entries[key] = (time.monotonic() + self._ttl, value)
+ self._entries.move_to_end(key)
+ while len(self._entries) > self._max:
+ self._entries.popitem(last=False)
+
+
+class MessageTelemetry:
+ """Reports `room_message_sent`, `agent_message_sent` and
+ `agent_message_received`.
+
+ The transport's `ParticipantMessageObserver`, and what the agent consumer
+ tells when a message is addressed to its agent. Does nothing at all — no
+ queue, no lookups — while telemetry is off.
+ """
+
+ def __init__(
+ self,
+ *,
+ telemetry: TelemetryService,
+ session_factory: async_sessionmaker[AsyncSession],
+ ) -> None:
+ self._telemetry = telemetry
+ self._session_factory = session_factory
+ self._queue: asyncio.Queue[ParticipantMessage | _AgentAddressed] = (
+ asyncio.Queue(maxsize=_QUEUE_SIZE)
+ )
+ self._worker: asyncio.Task[None] | None = None
+ self._closed = False
+ self._rooms: _TtlCache[tuple[str, str], _RoomFacts] = _TtlCache(
+ _CACHE_TTL_SECONDS, _CACHE_MAX_ENTRIES
+ )
+ self._sender_kinds: _TtlCache[tuple[str, str], str] = _TtlCache(
+ _CACHE_TTL_SECONDS, _CACHE_MAX_ENTRIES
+ )
+ self._agent_runtimes: _TtlCache[tuple[str, str], str] = _TtlCache(
+ _CACHE_TTL_SECONDS, _CACHE_MAX_ENTRIES
+ )
+ # Keys whose lookup failed, as `(lookup, tenant_id, id)`, until they
+ # may be tried again.
+ self._failed_lookups: _TtlCache[tuple[str, str, str], bool] = _TtlCache(
+ _LOOKUP_BACKOFF_SECONDS, _CACHE_MAX_ENTRIES
+ )
+ self._failures_in_a_row = 0
+ self._all_lookups_resume_at: float | None = None
+ # Taken off the queue and not yet reported. Not `qsize`, which leaves
+ # out the one the worker is cancelled in the middle of.
+ self._in_hand = 0
+ self._drops = WarningThrottle(_WARNING_INTERVAL_SECONDS)
+ self._late = WarningThrottle(_WARNING_INTERVAL_SECONDS)
+ self._report_failures = WarningThrottle(_WARNING_INTERVAL_SECONDS)
+ self._lookup_failures = WarningThrottle(_WARNING_INTERVAL_SECONDS)
+ self._lookup_pauses = WarningThrottle(_WARNING_INTERVAL_SECONDS)
+ self._lookup_misses = WarningThrottle(_WARNING_INTERVAL_SECONDS)
+
+ @property
+ def enabled(self) -> bool:
+ """Whether anything handed over is reported. A caller with work to do
+ only for telemetry can skip it while this is false."""
+ return self._telemetry.enabled
+
+ # ── Called on the sender's path ──────────────────────────────────────────
+
+ def observe(self, message: ParticipantMessage) -> None:
+ """A participant said something in a room."""
+ self._enqueue(message)
+
+ def agent_addressed(
+ self,
+ *,
+ tenant_id: str,
+ room_id: str,
+ sender_transport_user_id: str,
+ from_platform: bool,
+ agent_metadata: dict | None,
+ agent_live: bool,
+ has_attachment: bool,
+ ) -> None:
+ """A message was addressed to an agent and let through its addressing
+ policy and budget. `agent_live` is whether the agent had a live session
+ for the room when it arrived.
+
+ The runtime is read from `agent_metadata` here rather than by the
+ worker, so the queue holds a value and not the agent's own dict.
+ """
+ self._enqueue(
+ _AgentAddressed(
+ tenant_id=tenant_id,
+ room_id=room_id,
+ sender_transport_user_id=sender_transport_user_id,
+ from_platform=from_platform,
+ known_agent_type=normalise_known_agent_type(agent_metadata),
+ agent_live=agent_live,
+ has_attachment=has_attachment,
+ )
+ )
+
+ def _enqueue(self, item: ParticipantMessage | _AgentAddressed) -> None:
+ if not self._telemetry.enabled:
+ return
+ if self._closed:
+ # Senders still run while shutdown drains, and what they say then
+ # is lost; counted, so a low figure for the last minute has a cause.
+ if (late := self._late.note()) is not None:
+ logger.warning(
+ "%d message event(s) arrived after message telemetry shut "
+ "down and were dropped.",
+ late,
+ )
+ return
+ if self._worker is None or self._worker.done():
+ # Before the put, so a worker that ended with the queue full is
+ # replaced rather than leaving every later event to be dropped.
+ # A fresh context: started from whichever sender enqueued first, it
+ # would otherwise carry that sender's tenant and log fields for its
+ # whole life.
+ self._worker = asyncio.get_running_loop().create_task(
+ self._run(), context=contextvars.Context()
+ )
+ try:
+ self._queue.put_nowait(item)
+ except asyncio.QueueFull:
+ if (dropped := self._drops.note()) is not None:
+ self._warn_dropped(dropped)
+
+ @staticmethod
+ def _warn_dropped(dropped: int) -> None:
+ logger.warning(
+ "Message telemetry is behind: %d event(s) dropped because %d were "
+ "already queued. Message counts in analytics will read low.",
+ dropped,
+ _QUEUE_SIZE,
+ )
+
+ # ── The worker ───────────────────────────────────────────────────────────
+
+ async def _run(self) -> None:
+ while True:
+ item = await self._queue.get()
+ self._in_hand += 1
+ try:
+ await self._report(item)
+ except Exception:
+ # One bad event must not stop the rest being reported, and a
+ # bug that breaks every event must not log once per message.
+ if (failures := self._report_failures.note()) is not None:
+ logger.error(
+ "Could not report a message event; %d event(s) dropped "
+ "this way since the last error.",
+ failures,
+ exc_info=True,
+ )
+ finally:
+ self._queue.task_done()
+ # Not in the `finally`: an event the shutdown cancels mid-report
+ # was never reported, and leaving it counted is what says so.
+ self._in_hand -= 1
+
+ async def _report(self, item: ParticipantMessage | _AgentAddressed) -> None:
+ if isinstance(item, _AgentAddressed):
+ await self._report_addressed(item)
+ else:
+ await self._report_message(item)
+
+ async def _report_message(self, message: ParticipantMessage) -> None:
+ room = await self._room_facts(message.tenant_id, message.room_id)
+ self._telemetry.emit(
+ "room_message_sent",
+ sender_kind=_ROLE_KIND.get(message.sender_role, "unknown"),
+ bridge_platform=room.bridge_platform,
+ channel_type=room.channel_type,
+ room_user_count=room.user_count,
+ room_agent_count=room.agent_count,
+ has_attachment=message.has_attachment,
+ in_thread=message.in_thread,
+ )
+ if message.sender_role != "agent":
+ return
+ self._telemetry.emit(
+ "agent_message_sent",
+ known_agent_type=await self._agent_runtime(
+ message.tenant_id, message.sender_client_id
+ ),
+ bridge_platform=room.bridge_platform,
+ channel_type=room.channel_type,
+ room_user_count=room.user_count,
+ has_attachment=message.has_attachment,
+ in_thread=message.in_thread,
+ )
+
+ async def _report_addressed(self, item: _AgentAddressed) -> None:
+ room = await self._room_facts(item.tenant_id, item.room_id)
+ sender_kind = (
+ "platform"
+ if item.from_platform
+ else await self._sender_kind(item.tenant_id, item.sender_transport_user_id)
+ )
+ self._telemetry.emit(
+ "agent_message_received",
+ sender_kind=sender_kind,
+ known_agent_type=item.known_agent_type,
+ bridge_platform=room.bridge_platform,
+ channel_type=room.channel_type,
+ has_attachment=item.has_attachment,
+ agent_live=item.agent_live,
+ )
+
+ async def aclose(self) -> None:
+ """Report what is already queued, briefly, then stop."""
+ self._closed = True
+ try:
+ if self._worker is not None:
+ try:
+ async with asyncio.timeout(_DRAIN_SECONDS):
+ await self._queue.join()
+ except TimeoutError:
+ pass
+ finally:
+ self._worker.cancel()
+ # `wait` rather than awaiting the task: it does not raise the
+ # worker's own cancellation, so nothing here has to suppress
+ # one, and a cancellation aimed at this caller — the shutdown
+ # timeout — still reaches it. Outside the `finally`, so a
+ # cancellation that lands during the drain is not followed by
+ # an unbounded wait.
+ await asyncio.wait({self._worker})
+ finally:
+ self._report_unsent()
+
+ def _report_unsent(self) -> None:
+ """Say what shutdown could not report. Run from a `finally`, so the
+ caller's timeout cutting `aclose` off still leaves the count."""
+ if unreported := self._queue.qsize() + self._in_hand:
+ logger.warning(
+ "%d message event(s) still queued at shutdown were dropped.",
+ unreported,
+ )
+ if dropped := self._drops.take_pending():
+ self._warn_dropped(dropped)
+ if late := self._late.take_pending():
+ logger.warning(
+ "%d more message event(s) arrived after message telemetry shut "
+ "down and were dropped.",
+ late,
+ )
+ if failures := self._report_failures.take_pending():
+ logger.error(
+ "%d more message event(s) could not be reported since the last "
+ "error and were dropped.",
+ failures,
+ )
+
+ # ── Lookups ──────────────────────────────────────────────────────────────
+
+ def _may_look_up(self, key: tuple[str, str, str]) -> bool:
+ """Whether `key` may be looked up now: neither it nor the database as a
+ whole failed within the last `_LOOKUP_BACKOFF_SECONDS`."""
+ if (
+ self._all_lookups_resume_at is not None
+ and time.monotonic() < self._all_lookups_resume_at
+ ):
+ return False
+ return self._failed_lookups.get(key) is None
+
+ def _lookup_answered(self) -> None:
+ """The database answered, whether or not it found anything."""
+ self._failures_in_a_row = 0
+
+ def _lookup_failed(self, key: tuple[str, str, str], what: str) -> None:
+ """Back off after a failed lookup, and say so at most once a minute.
+
+ Called from inside the `except` block, so the warning carries the
+ traceback of the failure that triggered it.
+ """
+ self._failed_lookups.put(key, True)
+ self._failures_in_a_row += 1
+ if (failures := self._lookup_failures.note()) is not None:
+ logger.warning(
+ "Message telemetry could not look up %s, so events report it as "
+ "unknown and it is not looked up again for %.0fs; %d lookup(s) "
+ "failed since the last warning.",
+ what,
+ _LOOKUP_BACKOFF_SECONDS,
+ failures,
+ exc_info=True,
+ )
+ # Not reset when it trips: while the database stays down, the first
+ # lookup after each pause fails and starts the next one at once.
+ if self._failures_in_a_row >= _FAILURES_TO_PAUSE_ALL:
+ self._all_lookups_resume_at = time.monotonic() + _LOOKUP_BACKOFF_SECONDS
+ if (pauses := self._lookup_pauses.note()) is not None:
+ logger.warning(
+ "Message telemetry lookups failed %d times in a row, so every "
+ "lookup pauses for %.0fs and events report the rooms and "
+ "senders they need as unknown; %d pause(s) since the last "
+ "warning.",
+ self._failures_in_a_row,
+ _LOOKUP_BACKOFF_SECONDS,
+ pauses,
+ )
+
+ def _lookup_missed(self, what: str) -> None:
+ if (misses := self._lookup_misses.note()) is not None:
+ logger.warning(
+ "Message telemetry found no %s, so events report it as unknown; "
+ "%d lookup(s) found nothing since the last warning.",
+ what,
+ misses,
+ )
+
+ async def _room_facts(self, tenant_id: str, room_id: str) -> _RoomFacts:
+ key = (tenant_id, room_id)
+ cached = self._rooms.get(key)
+ if cached is not None:
+ return cached
+ failure_key = ("room", tenant_id, room_id)
+ if not self._may_look_up(failure_key):
+ return _UNKNOWN_ROOM
+ try:
+ facts = await self._load_room_facts(tenant_id, room_id)
+ except Exception:
+ self._lookup_failed(failure_key, f"room {room_id}")
+ return _UNKNOWN_ROOM
+ self._lookup_answered()
+ if facts is None:
+ self._lookup_missed(f"room {room_id}")
+ self._rooms.put(key, _UNKNOWN_ROOM)
+ return _UNKNOWN_ROOM
+ self._rooms.put(key, facts)
+ return facts
+
+ async def _load_room_facts(self, tenant_id: str, room_id: str) -> _RoomFacts | None:
+ async with tenant_session(self._session_factory, tenant_id) as session:
+ row = (
+ await session.execute(
+ select(Room.channel_type, CollaborationBridge.type)
+ .select_from(Room)
+ .outerjoin(
+ CollaborationBridge,
+ and_(
+ CollaborationBridge.id == Room.bridge_id,
+ CollaborationBridge.tenant_id == tenant_id,
+ ),
+ )
+ .where(Room.tenant_id == tenant_id, Room.id == room_id)
+ )
+ ).one_or_none()
+ if row is None:
+ return None
+ members = await session.execute(
+ select(Client.type, func.count())
+ .select_from(ClientRoom)
+ .join(Client, Client.id == ClientRoom.client_id)
+ .where(
+ ClientRoom.tenant_id == tenant_id,
+ ClientRoom.room_id == room_id,
+ Client.tenant_id == tenant_id,
+ )
+ .group_by(Client.type)
+ )
+ by_type = {client_type: int(count) for client_type, count in members.all()}
+ channel_type, platform = row
+ return _RoomFacts(
+ bridge_platform=normalise_platform(platform),
+ channel_type=normalise_channel_type(channel_type),
+ user_count=by_type.get(HUMAN_CLIENT_TYPE, 0),
+ agent_count=by_type.get(AGENT_CLIENT_TYPE, 0),
+ )
+
+ async def _sender_kind(self, tenant_id: str, transport_user_id: str) -> str:
+ key = (tenant_id, transport_user_id)
+ cached = self._sender_kinds.get(key)
+ if cached is not None:
+ return cached
+ failure_key = ("sender", tenant_id, transport_user_id)
+ if not self._may_look_up(failure_key):
+ return "unknown"
+ try:
+ async with tenant_session(self._session_factory, tenant_id) as session:
+ client_type = (
+ await session.execute(
+ select(Client.type).where(
+ Client.tenant_id == tenant_id,
+ Client.transport_user_id == transport_user_id,
+ )
+ )
+ ).scalar_one_or_none()
+ except Exception:
+ self._lookup_failed(failure_key, "a message sender")
+ return "unknown"
+ self._lookup_answered()
+ if client_type is None:
+ self._lookup_missed("client for a message sender")
+ self._sender_kinds.put(key, "unknown")
+ return "unknown"
+ kind = _CLIENT_TYPE_KIND.get(client_type, "unknown")
+ self._sender_kinds.put(key, kind)
+ return kind
+
+ async def _agent_runtime(self, tenant_id: str, client_id: str) -> str:
+ key = (tenant_id, client_id)
+ cached = self._agent_runtimes.get(key)
+ if cached is not None:
+ return cached
+ failure_key = ("agent", tenant_id, client_id)
+ if not self._may_look_up(failure_key):
+ return _UNKNOWN_RUNTIME
+ try:
+ async with tenant_session(self._session_factory, tenant_id) as session:
+ result = await session.execute(
+ select(Agent.metadata_).where(
+ Agent.tenant_id == tenant_id, Agent.client_id == client_id
+ )
+ )
+ row = result.one_or_none()
+ except Exception:
+ self._lookup_failed(failure_key, "an agent's runtime")
+ return _UNKNOWN_RUNTIME
+ self._lookup_answered()
+ if row is None:
+ self._lookup_missed(f"agent for client {client_id}")
+ self._agent_runtimes.put(key, _UNKNOWN_RUNTIME)
+ return _UNKNOWN_RUNTIME
+ runtime = normalise_known_agent_type(row[0])
+ self._agent_runtimes.put(key, runtime)
+ return runtime
diff --git a/core/switch_core/telemetry/service.py b/core/switch_core/telemetry/service.py
index 2fcc125fc..b0d440d6a 100644
--- a/core/switch_core/telemetry/service.py
+++ b/core/switch_core/telemetry/service.py
@@ -1,8 +1,8 @@
"""The one call every site uses to report an event.
`emit` is fire-and-forget and synchronous to call: creating a room must not
-wait on a relay. It validates against the catalogue first, then hands the send
-to a background task.
+wait on a relay. It validates against the catalogue first, then hands the
+record to the sink, which only buffers it.
The asymmetry is deliberate. **A bad event raises** — it is a programming error
and silence would defeat the catalogue. **A failed send does not** — it is
@@ -71,7 +71,6 @@ class TelemetryService:
self._resource["service.version"] = version
if environment:
self._resource["deployment.environment"] = environment
- self._tasks: set[asyncio.Task[None]] = set()
@property
def enabled(self) -> bool:
@@ -127,40 +126,27 @@ class TelemetryService:
def _dispatch(self, record: TelemetryRecord) -> None:
try:
- loop = asyncio.get_running_loop()
+ asyncio.get_running_loop()
except RuntimeError:
# A management command or a sync helper; not its job to have one.
logger.debug(
"Telemetry event %s not sent: no running event loop.", record.name
)
return
-
- task = loop.create_task(self._send(record))
- # Held for the lifetime of the send. An un-referenced task can be
- # garbage-collected mid-flight, which drops the event silently and is
- # exactly the kind of bug telemetry code is bad at revealing.
- self._tasks.add(task)
- task.add_done_callback(self._tasks.discard)
-
- async def _send(self, record: TelemetryRecord) -> None:
try:
- await self._sink.send(record)
- except asyncio.CancelledError:
- raise
+ self._sink.send(record)
except Exception:
- # A bug in the sink, not a bad relay — but this is a bare task, so
- # letting it out only surfaces at some later collection.
+ # A bug in the sink, not a bad relay. `emit` raises only for a
+ # malformed event, so this must not reach the caller either.
logger.exception("Telemetry sink failed on event %s", record.name)
async def aclose(self) -> None:
- """Let in-flight sends finish, then close the sink.
+ """Close the sink, which posts what it still holds.
Shutdown is the one time waiting is right: the events worth losing
least are the ones emitted just before the process goes away, and the
- sink's own timeout already bounds how long this can take.
+ caller's timeout bounds how long this can take.
"""
- if self._tasks:
- await asyncio.gather(*tuple(self._tasks), return_exceptions=True)
await self._sink.aclose()
diff --git a/core/switch_core/telemetry/setup.py b/core/switch_core/telemetry/setup.py
index 3b065c8f6..5c348dfba 100644
--- a/core/switch_core/telemetry/setup.py
+++ b/core/switch_core/telemetry/setup.py
@@ -22,6 +22,14 @@ from switch_core.telemetry.sink import NullSink, OtlpRelaySink, TelemetrySink
logger = logging.getLogger(__name__)
+# How the relay sink batches. Five seconds keeps an event's delay well inside
+# what analytics cares about; two hundred records is a request of a few tens of
+# kilobytes. The buffer holds roughly a minute of a very busy server, so a
+# relay that is down for longer loses events rather than growing memory.
+_FLUSH_INTERVAL_SECONDS = 5.0
+_MAX_BATCH = 200
+_MAX_BUFFERED = 10_000
+
async def build_telemetry(
config: SwitchConfig,
@@ -92,7 +100,10 @@ async def build_telemetry(
timeout_seconds=config.telemetry_timeout_seconds,
headers={},
client=http_client,
- )
+ ),
+ flush_interval_seconds=_FLUSH_INTERVAL_SECONDS,
+ max_batch=_MAX_BATCH,
+ max_buffered=_MAX_BUFFERED,
)
logger.info(
"Product telemetry is ON: usage counts and timings are reported to "
diff --git a/core/switch_core/telemetry/sink.py b/core/switch_core/telemetry/sink.py
index 932a6518f..b063e0a72 100644
--- a/core/switch_core/telemetry/sink.py
+++ b/core/switch_core/telemetry/sink.py
@@ -9,19 +9,24 @@ lets a reporting problem reach the caller.
from __future__ import annotations
+import asyncio
+import contextvars
import logging
-from collections.abc import Mapping
+from collections import deque
+from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from typing import Any, Protocol
from switch_core.observability.otlp import (
LogRecord,
OtlpClient,
+ OtlpPartialRejection,
OtlpResource,
OtlpSendError,
build_logs_payload,
otlp_attributes,
)
+from switch_core.observability.throttle import WarningThrottle
from switch_core.telemetry.catalogue import PropertyValue
logger = logging.getLogger(__name__)
@@ -31,6 +36,14 @@ logger = logging.getLogger(__name__)
# receivers as an error.
_SEVERITY_INFO = 9
+_DROP_WARNING_INTERVAL_SECONDS = 60.0
+
+# How many batches shutdown posts at once. One at a time, a slow request
+# already in flight would hold every batch behind it until the budget ran out;
+# a handful rather than the fifty a full buffer makes, because the relay
+# rate-limits on the sender's address.
+_SHUTDOWN_CONCURRENCY = 4
+
@dataclass(frozen=True)
class TelemetryRecord:
@@ -48,9 +61,11 @@ class TelemetryRecord:
class TelemetrySink(Protocol):
- """The seam. One method, and it never raises."""
+ """The seam. `send` hands a record over and returns at once: it never
+ raises and never waits on the network, so it is safe to call once per
+ message. Called with an event loop running."""
- async def send(self, record: TelemetryRecord) -> None: ...
+ def send(self, record: TelemetryRecord) -> None: ...
async def aclose(self) -> None: ...
@@ -59,7 +74,7 @@ class NullSink:
"""Drops everything. Off is a sink that discards, so no call site has to
ask whether telemetry is on."""
- async def send(self, record: TelemetryRecord) -> None:
+ def send(self, record: TelemetryRecord) -> None:
return None
async def aclose(self) -> None:
@@ -67,7 +82,7 @@ class NullSink:
class OtlpRelaySink:
- """Posts one OTLP log record per event to the relay.
+ """Posts events to the relay as OTLP log records, a batch at a time.
Log records, not metrics: the relay routes on them and the analytics
exporter reads them as events. A metric would be dropped without complaint.
@@ -78,45 +93,223 @@ class OtlpRelaySink:
and then discarded. That is the easiest way to believe this works when it
does not.
- No batching and no retry, matching the operational exporter.
+ **Batched**, because the message events fire once per message: a busy
+ server posting one request per event would be a load the relay was never
+ sized for, and would trip the rate limit it keys on the sender's address.
+ `send` only buffers; one task posts every `flush_interval_seconds`, or as
+ soon as `max_batch` records are waiting, and `aclose` posts what is left.
+ A flush posts what was buffered when it began and no more: what arrives
+ while it is posting waits for the next interval or a full batch, or a
+ steady trickle would become a request per event again.
+
+ No retry, matching the operational exporter, and the buffer is bounded: a
+ relay that stops answering costs the events it misses, never memory.
+
+ **Shutdown** posts what is left several batches at once, beside any request
+ already in flight rather than behind it, and stops at the first batch the
+ relay refuses. Whatever the caller's timeout cuts off, buffered or
+ abandoned mid-request, is counted and logged.
"""
- def __init__(self, *, client: OtlpClient) -> None:
+ def __init__(
+ self,
+ *,
+ client: OtlpClient,
+ flush_interval_seconds: float,
+ max_batch: int,
+ max_buffered: int,
+ ) -> None:
self._client = client
-
- async def send(self, record: TelemetryRecord) -> None:
- payload = build_logs_payload(
- [
- LogRecord(
- # Datadog renders this as the log message, and a blank one
- # makes the event unreadable there.
- body=record.name,
- severity_text="INFO",
- severity_number=_SEVERITY_INFO,
- time_nanos=record.timestamp_ns,
- attributes={"event.name": record.name, **record.properties},
+ self._flush_interval = flush_interval_seconds
+ self._max_batch = max_batch
+ self._max_buffered = max_buffered
+ self._buffer: list[TelemetryRecord] = []
+ # Taken from the buffer, waiting for a post.
+ self._batches: deque[list[TelemetryRecord]] = deque()
+ # Records in posts that have not finished.
+ self._posting = 0
+ self._wake = asyncio.Event()
+ self._flusher: asyncio.Task[None] | None = None
+ self._closing = False
+ self._refused_at_shutdown = False
+ self._drops = WarningThrottle(_DROP_WARNING_INTERVAL_SECONDS)
+ self._late = WarningThrottle(_DROP_WARNING_INTERVAL_SECONDS)
+
+ def send(self, record: TelemetryRecord) -> None:
+ if self._closing:
+ # Shutdown has already taken the final batch; posting this one
+ # would race the client being closed.
+ if (late := self._late.note()) is not None:
+ logger.warning(
+ "%d telemetry event(s), the latest %s, arrived after "
+ "shutdown began and were dropped.",
+ late,
+ record.name,
)
- ],
- _resource_from(record),
+ return
+ if self._flusher is None or self._flusher.done():
+ # Before the capacity check, so a flusher that ended with the
+ # buffer full is replaced rather than leaving every later event to
+ # be dropped. A fresh context, so its warnings do not carry the log
+ # fields of whichever request happened to emit first.
+ self._flusher = asyncio.get_running_loop().create_task(
+ self._run(), context=contextvars.Context()
+ )
+ if len(self._buffer) >= self._max_buffered:
+ if (dropped := self._drops.note()) is not None:
+ self._warn_dropped(dropped)
+ return
+ self._buffer.append(record)
+ if len(self._buffer) >= self._max_batch:
+ self._wake.set()
+
+ def _warn_dropped(self, dropped: int) -> None:
+ logger.warning(
+ "Telemetry buffer is full (%d events waiting on the relay): %d "
+ "event(s) dropped since the last warning.",
+ self._max_buffered,
+ dropped,
)
- _add_event_name(payload, record.name)
- _add_relay_context(payload, record.resource)
- try:
- await self._client.post("logs", payload)
- except OtlpSendError as exc:
- # Switch has to keep working while analytics is down.
- logger.warning(
- "Telemetry event %s was not sent: %s. The event is dropped; "
- "there is no retry.",
- record.name,
- exc,
+ async def _run(self) -> None:
+ while True:
+ try:
+ async with asyncio.timeout(self._flush_interval):
+ await self._wake.wait()
+ except TimeoutError:
+ pass
+ self._wake.clear()
+ if self._closing:
+ # `aclose` has taken the buffer and posts it.
+ return
+ self._take_buffer()
+ await self._post_waiting()
+
+ def _take_buffer(self) -> None:
+ pending, self._buffer = self._buffer, []
+ for start in range(0, len(pending), self._max_batch):
+ self._batches.append(pending[start : start + self._max_batch])
+
+ async def _post_waiting(self) -> None:
+ """Post batches until none are waiting.
+
+ Outside shutdown only the flusher takes from the buffer, and only as a
+ flush begins, so this posts what that flush took and no more.
+ """
+ while self._batches and not self._refused_at_shutdown:
+ batch = self._batches.popleft()
+ self._posting += len(batch)
+ try:
+ await self._post(batch)
+ except Exception:
+ # A relay failure is already handled in `_post`; anything
+ # reaching here is a bug in building the batch. Caught per
+ # batch, so it costs neither the batches after it nor the task
+ # every later event depends on.
+ logger.exception(
+ "Telemetry batch of %d event(s) could not be built; it is dropped.",
+ len(batch),
+ )
+ # Not in a `finally`: a post the shutdown timeout cancels did not
+ # send its records, and leaving them counted here is what reports
+ # them as unsent.
+ self._posting -= len(batch)
+
+ async def _post(self, batch: Sequence[TelemetryRecord]) -> None:
+ # Every record a service emits carries the same resource, so this is
+ # one payload in practice; grouping keeps two services sharing a sink
+ # from being reported under each other's identity.
+ by_resource: dict[tuple[tuple[str, str], ...], list[TelemetryRecord]] = {}
+ for record in batch:
+ key = tuple(sorted(record.resource.items()))
+ by_resource.setdefault(key, []).append(record)
+
+ for records in by_resource.values():
+ payload = build_logs_payload(
+ [
+ LogRecord(
+ # Datadog renders this as the log message, and a blank
+ # one makes the event unreadable there.
+ body=record.name,
+ severity_text="INFO",
+ severity_number=_SEVERITY_INFO,
+ time_nanos=record.timestamp_ns,
+ attributes={"event.name": record.name, **record.properties},
+ )
+ for record in records
+ ],
+ _resource_from(records[0]),
)
+ _add_event_names(payload, [record.name for record in records])
+ _add_relay_context(payload, records[0].resource)
+
+ names = ", ".join(sorted({record.name for record in records}))
+ try:
+ await self._client.post("logs", payload)
+ except OtlpPartialRejection as exc:
+ logger.warning(
+ "The relay rejected %d of %d telemetry event(s) in a batch "
+ "(%s): %s. The rest were accepted; there is no retry.",
+ exc.rejected,
+ len(records),
+ names,
+ exc.reason,
+ )
+ except OtlpSendError as exc:
+ # Switch has to keep working while analytics is down.
+ logger.warning(
+ "Telemetry batch of %d event(s) (%s) was not sent: %s. The "
+ "batch is dropped; there is no retry.",
+ len(records),
+ names,
+ exc,
+ )
+ if self._closing:
+ # Feeding the rest to a relay that has just refused would
+ # spend the shutdown budget losing them more slowly.
+ self._refused_at_shutdown = True
async def aclose(self) -> None:
- # `telemetry/setup.py` opens the client and `main._drain_telemetry`
- # closes it, under a timeout shutdown depends on.
- return None
+ """Post everything still buffered, under the caller's timeout.
+
+ `telemetry/setup.py` opens the HTTP client and `main._drain_telemetry`
+ closes it, under a timeout shutdown depends on. The flusher is awaited
+ with the rest, so that timeout cancels it along with this call rather
+ than leaving it posting.
+ """
+ self._closing = True
+ self._wake.set()
+ try:
+ self._take_buffer()
+ posts: list[asyncio.Future[None]] = [
+ asyncio.ensure_future(self._post_waiting())
+ for _ in range(_SHUTDOWN_CONCURRENCY)
+ ]
+ if self._flusher is not None and not self._flusher.done():
+ posts.append(self._flusher)
+ await asyncio.gather(*posts)
+ finally:
+ self._report_unsent()
+
+ def _report_unsent(self) -> None:
+ """Say what shutdown could not post. Run from a `finally`: the caller's
+ timeout cutting `aclose` off is the case that loses the most."""
+ if unsent := self._posting + sum(len(batch) for batch in self._batches):
+ logger.warning(
+ "%d telemetry event(s) were never sent: %s.",
+ unsent,
+ "the relay refused a batch at shutdown, so the rest were not tried"
+ if self._refused_at_shutdown
+ else "shutdown ran out of time before they were posted",
+ )
+ if dropped := self._drops.take_pending():
+ self._warn_dropped(dropped)
+ if late := self._late.take_pending():
+ logger.warning(
+ "%d more telemetry event(s) arrived after shutdown began and "
+ "were dropped.",
+ late,
+ )
def _resource_from(record: TelemetryRecord) -> OtlpResource:
@@ -146,10 +339,15 @@ def _add_relay_context(payload: dict[str, Any], resource: Mapping[str, str]) ->
resource_log["resource"]["attributes"].extend(otlp_attributes(context))
-def _add_event_name(payload: dict[str, Any], name: str) -> None:
- """Set the record's own `eventName` field, which `build_logs_payload` does
- not: an operational log line has no event name."""
- for resource_log in payload["resourceLogs"]:
- for scope_log in resource_log["scopeLogs"]:
- for record in scope_log["logRecords"]:
- record["eventName"] = name
+def _add_event_names(payload: dict[str, Any], names: Sequence[str]) -> None:
+ """Set each record's own `eventName` field, which `build_logs_payload` does
+ not: an operational log line has no event name. `names` is in the order
+ the records were handed to the encoder, which keeps it."""
+ encoded = [
+ record
+ for resource_log in payload["resourceLogs"]
+ for scope_log in resource_log["scopeLogs"]
+ for record in scope_log["logRecords"]
+ ]
+ for record, name in zip(encoded, names, strict=True):
+ record["eventName"] = name
diff --git a/core/switch_core/telemetry/snapshot.py b/core/switch_core/telemetry/snapshot.py
index 85238810e..998b6a116 100644
--- a/core/switch_core/telemetry/snapshot.py
+++ b/core/switch_core/telemetry/snapshot.py
@@ -31,6 +31,7 @@ from sqlalchemy import (
exists,
func,
select,
+ union,
union_all,
)
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
@@ -131,6 +132,12 @@ class UsageCounts:
chat_identity_count: int = 0
chat_identity_active_1d: int = 0
chat_identity_active_7d: int = 0
+ # The other tiers, as chat identities like `chat_identity_active_*`:
+ # anyone who said anything in any room, and anyone Switch has seen in a
+ # room — see `_activity_tiers` for what that can and cannot see.
+ chat_identity_posted_1d: int = 0
+ chat_identity_posted_7d: int = 0
+ chat_identity_in_room_count: int = 0
room_count: int = 0
room_agent_created_count: int = 0
room_system_created_count: int = 0
@@ -180,6 +187,9 @@ class UsageCounts:
"chat_identity_count": self.chat_identity_count,
"chat_identity_active_1d": self.chat_identity_active_1d,
"chat_identity_active_7d": self.chat_identity_active_7d,
+ "chat_identity_posted_1d": self.chat_identity_posted_1d,
+ "chat_identity_posted_7d": self.chat_identity_posted_7d,
+ "chat_identity_in_room_count": self.chat_identity_in_room_count,
"room_count": self.room_count,
"room_agent_created_count": self.room_agent_created_count,
"room_system_created_count": self.room_system_created_count,
@@ -387,6 +397,66 @@ def _active_humans(tenant_id: str, since: datetime) -> Select[tuple[str | None]]
)
+def _activity_tiers(
+ tenant_id: str, day_ago: datetime, week_ago: datetime
+) -> Select[tuple[int, int, int]]:
+ """Distinct human clients who said anything in any room in the last day
+ and week, and those Switch has seen in a room: a member of a live room
+ now, or anyone who posted in the last week.
+
+ The posters are `_active_humans` without the agent condition: someone
+ talking only to other people is not using an agent, but is not passive
+ either. They are folded into the room count so the tiers nest: someone who
+ posted this week and then left, or whose room was archived, is no longer a
+ member of a live room but has still been seen. Rooms of every origin, like
+ the activity counts: a person in an adopted channel is in Switch whether
+ or not anybody created that room here.
+
+ Not a channel's whole audience. A person gets a membership row only when
+ they post, are added by name, or join after the channel was adopted;
+ nothing records the members a channel already had, so someone who has
+ only ever read is not here.
+
+ One statement over one read of the week's messages, so the three come from
+ the same snapshot and nest exactly, and the scan is not repeated per tier.
+ """
+ posters = (
+ select(
+ Message.sender_client_id.label("client_id"),
+ func.max(Message.sent_at).label("last_posted_at"),
+ )
+ .join(Client, Client.id == Message.sender_client_id)
+ .where(
+ Message.tenant_id == tenant_id,
+ Message.sent_at >= week_ago,
+ *_human_message_conditions(tenant_id),
+ )
+ .group_by(Message.sender_client_id)
+ .cte("posters")
+ )
+ members = (
+ select(ClientRoom.client_id)
+ .join(Client, Client.id == ClientRoom.client_id)
+ .join(Room, Room.id == ClientRoom.room_id)
+ .where(
+ ClientRoom.tenant_id == tenant_id,
+ Client.tenant_id == tenant_id,
+ Client.type == HUMAN_CLIENT_TYPE,
+ Room.tenant_id == tenant_id,
+ Room.archived_at.is_(None),
+ )
+ )
+ seen = union(members, select(posters.c.client_id)).subquery("seen")
+ return select(
+ select(func.count())
+ .select_from(posters)
+ .where(posters.c.last_posted_at >= day_ago)
+ .scalar_subquery(),
+ select(func.count()).select_from(posters).scalar_subquery(),
+ select(func.count()).select_from(seen).scalar_subquery(),
+ )
+
+
async def _count(session: AsyncSession, query: Select[Any]) -> int:
result = await session.execute(select(func.count()).select_from(query.subquery()))
return int(result.scalar_one())
@@ -431,6 +501,12 @@ async def collect_tenant_counts(
counts.chat_identity_active_7d += await _count(
session, _active_humans(tenant_id, week_ago)
)
+ posted_1d, posted_7d, in_room = (
+ await session.execute(_activity_tiers(tenant_id, day_ago, week_ago))
+ ).one()
+ counts.chat_identity_posted_1d += posted_1d
+ counts.chat_identity_posted_7d += posted_7d
+ counts.chat_identity_in_room_count += in_room
counts.room_active_1d += await _count(
session, _human_interaction(tenant_id, day_ago)
)
diff --git a/core/switch_core/transport/observer.py b/core/switch_core/transport/observer.py
new file mode 100644
index 000000000..f7f2aefe7
--- /dev/null
+++ b/core/switch_core/transport/observer.py
@@ -0,0 +1,48 @@
+"""What the transport tells the rest of Switch about a message once it is sent.
+
+The transport is the one place every participant's message passes through, so
+it is where "a message was said in a room" can be observed without each writer
+having to remember to report it. It knows nothing about who is listening: it
+hands over the facts it already holds, after the commit, and carries on.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Protocol
+
+
+@dataclass(frozen=True)
+class ParticipantMessage:
+ """A message a participant chose to say, as the transport saw it written.
+
+ The same population the tenant is metered for: notices Switch posts on a
+ participant's behalf are not in it. A multi-file post is reported once.
+ """
+
+ tenant_id: str
+ room_id: str
+ sender_client_id: str
+ #: The writer's `ActorRole`: `human`, `agent`, `system` or `bridge`.
+ sender_role: str
+ has_attachment: bool
+ in_thread: bool
+
+
+class ParticipantMessageObserver(Protocol):
+ """Told about every participant message after it is committed.
+
+ Called on the sender's path, so it must return at once and never raise:
+ the message is already sent, and a failure here would tell the sender
+ otherwise.
+ """
+
+ def observe(self, message: ParticipantMessage) -> None: ...
+
+
+class IgnoreParticipantMessages:
+ """An observer that does nothing, for transports built where no one is
+ listening — tooling and tests."""
+
+ def observe(self, message: ParticipantMessage) -> None:
+ return None
diff --git a/core/switch_core/transport/postgres.py b/core/switch_core/transport/postgres.py
index 0ca83ce15..95b5c1a64 100644
--- a/core/switch_core/transport/postgres.py
+++ b/core/switch_core/transport/postgres.py
@@ -36,6 +36,7 @@ import uuid
from datetime import UTC, datetime
from typing import TYPE_CHECKING, Any
+from switch_core.attachments import parse_attachment_group
from switch_core.db.models import (
ClientRoom,
MediaBlob,
@@ -51,13 +52,19 @@ from switch_core.observability.catalogue import (
DELIVERY_LAG,
MESSAGES_DELIVERED,
MESSAGES_SENT,
+ OBSERVER_FAILURES,
SEND_FAILURES,
)
from switch_core.observability.metrics import metrics
+from switch_core.observability.throttle import WarningThrottle
from switch_core.tenant_context import no_tenant, tenant_scope
from switch_core.transport.content import media_content, message_content
from switch_core.transport.ephemeral import EphemeralBus
from switch_core.transport.invites import InviteBus
+from switch_core.transport.observer import (
+ ParticipantMessage,
+ ParticipantMessageObserver,
+)
from switch_core.transport.port import Handler, TransportHandlers
from switch_core.transport.room_cache import DELIVERY_PAGE
from switch_core.transport.stored import to_inbound
@@ -99,6 +106,10 @@ _MEDIA_MSGTYPES = frozenset({"m.image", "m.file", "m.video", "m.audio"})
# a tenant for them would bill it for how Switch works.
_METERED_KINDS = frozenset({"message", "media"})
+# One for the process rather than one per transport: every client has a
+# transport, and an observer that breaks breaks for all of them.
+_observer_failures = WarningThrottle(60.0)
+
def _sent_kind(event_type: str, content: dict[str, object]) -> str:
"""A bounded label for what was sent.
@@ -175,6 +186,7 @@ class PostgresTransport:
invites: InviteBus,
ephemeral: EphemeralBus,
room_cache: RoomDeliveryCache,
+ message_observer: ParticipantMessageObserver,
) -> None:
self.user_id = user_id
self.client_id = client_id
@@ -209,6 +221,7 @@ class PostgresTransport:
# When this client was last woken for each room, on the cache's clock.
# A shared read only counts for this client if it started after this.
self._woken: dict[str, int] = {}
+ self._message_observer = message_observer
self._handlers = TransportHandlers()
# Per-room delivery position, and the transport-side id to hand back
# to a handler. Both are keyed by the Switch room id, which is what
@@ -800,8 +813,47 @@ class PostgresTransport:
)
raise
metrics().increment(MESSAGES_SENT, {"kind": kind, "actor": self._actor_role})
+ if metered and kind in _METERED_KINDS:
+ self._observe(tenant_id, room_id, kind, content)
return result
+ def _observe(
+ self, tenant_id: str, room_id: str, kind: str, content: dict[str, object]
+ ) -> None:
+ """Tell the observer a participant said something.
+
+ After the commit, so it never hears of a message that was rolled back.
+ Guarded because the row is already committed: an exception leaving
+ `_send` now would tell the sender that a message in the room was not
+ sent, and its retry would post it twice. Counted, because a guard that
+ only logs leaves nothing to alert on.
+ """
+ group = parse_attachment_group(content)
+ if group is not None and group[1] != 0:
+ # One post with several files is one message; its first part
+ # stands for it.
+ return
+ try:
+ self._message_observer.observe(
+ ParticipantMessage(
+ tenant_id=tenant_id,
+ room_id=room_id,
+ sender_client_id=self.client_id,
+ sender_role=self._actor_role,
+ has_attachment=kind == "media",
+ in_thread=thread_root_of(content) is not None,
+ )
+ )
+ except Exception:
+ metrics().increment(OBSERVER_FAILURES, {"actor": self._actor_role})
+ if (failures := _observer_failures.note()) is not None:
+ logger.error(
+ "Message observer failed; the message itself was sent. %d "
+ "failure(s) since the last error.",
+ failures,
+ exc_info=True,
+ )
+
async def set_typing(self, room_id: str, is_typing: bool) -> None:
"""Not carried. Composing state is presence, and nothing consumes it."""
diff --git a/core/tests/benchmarks/server.py b/core/tests/benchmarks/server.py
index b40d90d78..3949daabb 100644
--- a/core/tests/benchmarks/server.py
+++ b/core/tests/benchmarks/server.py
@@ -70,6 +70,9 @@ from switch_core.session_activity.listener import AgentSessionActivityListener
from switch_core.session_activity.outcomes import ApprovalOutcomes
from switch_core.session_activity.service import AgentSessionActivityService, SwitchUser
from switch_core.sessions.contract import ApprovalResult
+from switch_core.telemetry.messages import MessageTelemetry
+from switch_core.telemetry.service import TelemetryService
+from switch_core.telemetry.sink import NullSink
from switch_core.tenant_context import bind_tenant_id, tenant_scope
from switch_core.transport.ephemeral import EphemeralBus
from switch_core.transport.invites import InviteBus
@@ -487,6 +490,22 @@ async def _serve(
),
)
+ # Telemetry off: these flows are not about reporting, but the
+ # message hook must still be wired for the clients to build.
+ message_telemetry = MessageTelemetry(
+ telemetry=TelemetryService(
+ sink=NullSink(),
+ enabled=False,
+ client_id="",
+ service_name="switch-core",
+ version=None,
+ environment=None,
+ telemetry_environment="dev",
+ telemetry_internal=True,
+ ),
+ session_factory=session_factory,
+ )
+
client_factory = ClientFactory(
client_store=session_env.client_store,
session_factory=session_factory,
@@ -498,6 +517,7 @@ async def _serve(
invites=invites,
ephemeral=ephemeral,
room_cache=room_cache,
+ message_observer=message_telemetry,
)
client_factory.register(
"agent",
@@ -515,6 +535,7 @@ async def _serve(
hosted_launch_store=HostedLaunchStore(),
connections=connections,
frontend_base_url=config.frontend_base_url,
+ message_telemetry=message_telemetry,
)
client_factory.register("user", HumanActor)
client_factory.register("bridge", Actor)
diff --git a/core/tests/integration/conftest.py b/core/tests/integration/conftest.py
index e0572de00..22fb9e274 100644
--- a/core/tests/integration/conftest.py
+++ b/core/tests/integration/conftest.py
@@ -85,6 +85,9 @@ from switch_core.messages.notify import MessageListener
from switch_core.provisioning import Provisioning
from switch_core.provisioning.postgres import PostgresProvisioning
from switch_core.room_service import RoomService
+from switch_core.telemetry.messages import MessageTelemetry
+from switch_core.telemetry.service import TelemetryService
+from switch_core.telemetry.sink import NullSink
from switch_core.tenant_context import tenant_scope
from switch_core.transport.ephemeral import EphemeralBus
from switch_core.transport.invites import InviteBus
@@ -567,6 +570,22 @@ async def harness(session_env: SessionEnv) -> AsyncIterator[Harness]:
),
)
+ # Telemetry off: these flows are not about reporting, but the
+ # message hook must still be wired for the clients to build.
+ message_telemetry = MessageTelemetry(
+ telemetry=TelemetryService(
+ sink=NullSink(),
+ enabled=False,
+ client_id="",
+ service_name="switch-core",
+ version=None,
+ environment=None,
+ telemetry_environment="dev",
+ telemetry_internal=True,
+ ),
+ session_factory=session_factory,
+ )
+
client_factory = ClientFactory(
client_store=session_env.client_store,
session_factory=session_factory,
@@ -579,6 +598,7 @@ async def harness(session_env: SessionEnv) -> AsyncIterator[Harness]:
invites=invites,
ephemeral=ephemeral,
room_cache=room_cache,
+ message_observer=message_telemetry,
)
client_factory.register(
"agent",
@@ -596,6 +616,7 @@ async def harness(session_env: SessionEnv) -> AsyncIterator[Harness]:
hosted_launch_store=HostedLaunchStore(),
connections=connections,
frontend_base_url=config.frontend_base_url,
+ message_telemetry=message_telemetry,
)
client_factory.register("user", HumanActor)
client_factory.register("bridge", Actor)
diff --git a/core/tests/switch_core/bridges/agent/api/test_events_endpoint_dispatch.py b/core/tests/switch_core/bridges/agent/api/test_events_endpoint_dispatch.py
index 23e7f5937..d979bc3e3 100644
--- a/core/tests/switch_core/bridges/agent/api/test_events_endpoint_dispatch.py
+++ b/core/tests/switch_core/bridges/agent/api/test_events_endpoint_dispatch.py
@@ -444,7 +444,7 @@ class _RecordingSink:
def __init__(self) -> None:
self.sent: list[TelemetryRecord] = []
- async def send(self, record: TelemetryRecord) -> None:
+ def send(self, record: TelemetryRecord) -> None:
self.sent.append(record)
async def aclose(self) -> None:
diff --git a/core/tests/switch_core/bridges/agent/protocol/test_controller_presence.py b/core/tests/switch_core/bridges/agent/protocol/test_controller_presence.py
index 58e91387b..6a9710c78 100644
--- a/core/tests/switch_core/bridges/agent/protocol/test_controller_presence.py
+++ b/core/tests/switch_core/bridges/agent/protocol/test_controller_presence.py
@@ -573,6 +573,7 @@ def _addressing_client(
ns._fresh_agent = _fresh_agent
ns._gate_addressed = _gate_addressed
ns._note_hosted_addressed = _not_hosted
+ ns._report_addressed = lambda *_a, **_k: None
ns._triggered_by_auto_reply = AgentConsumer._triggered_by_auto_reply
ns.actor = SimpleNamespace(send_message=sent)
ns._event_buffer = SimpleNamespace(
diff --git a/core/tests/switch_core/clients/test_agent_consumer_attachment_groups.py b/core/tests/switch_core/clients/test_agent_consumer_attachment_groups.py
index b6306a901..1f8f041c4 100644
--- a/core/tests/switch_core/clients/test_agent_consumer_attachment_groups.py
+++ b/core/tests/switch_core/clients/test_agent_consumer_attachment_groups.py
@@ -6,6 +6,8 @@ from types import SimpleNamespace
from typing import Any
from unittest.mock import AsyncMock
+import pytest
+
import switch_core.clients.agent_consumer as ac
from switch_core.clients.agent_consumer import AgentConsumer, _GateOutcome
from switch_core.clients.room_meta import RoomMeta
@@ -54,6 +56,15 @@ async def _session_factory(): # type: ignore[no-untyped-def]
yield object()
+class _FakeMessageTelemetry:
+ def __init__(self) -> None:
+ self.enabled = True
+ self.addressed: list[dict[str, Any]] = []
+
+ def agent_addressed(self, **kwargs: Any) -> None:
+ self.addressed.append(kwargs)
+
+
class _FakeQueue:
def __init__(self) -> None:
self.events: list[Any] = []
@@ -84,20 +95,30 @@ def _fake_client() -> SimpleNamespace:
) -> _GateOutcome:
return _GateOutcome(addressed=True, refusal=None)
+ async def _is_available(_session: Any, _agent: Any, _room_id: str) -> bool:
+ ns.availability_checks += 1
+ return True
+
ns = SimpleNamespace(
- agent=SimpleNamespace(id="agent-1", name="agent-a"),
+ agent=SimpleNamespace(id="agent-1", name="agent-a", metadata_=None),
+ tenant_id="tenant-1",
session_factory=_session_factory,
_event_buffer=queue,
+ _message_telemetry=_FakeMessageTelemetry(),
+ _triggered_by_auto_reply=AgentConsumer._triggered_by_auto_reply,
_attachment_groups={},
_attachment_group_timers={},
_resolve_room_meta=_resolve_room_meta,
_addressed=_addressed,
_fresh_agent=_fresh_agent,
_gate_addressed=_gate_addressed,
+ _is_available=_is_available,
+ availability_checks=0,
queue=queue,
_note_hosted_addressed=AsyncMock(return_value=None),
)
ns._emit_media = AgentConsumer._emit_media.__get__(ns)
+ ns._report_addressed = AgentConsumer._report_addressed.__get__(ns)
ns._schedule_attachment_group_flush = (
AgentConsumer._schedule_attachment_group_flush.__get__(ns)
)
@@ -440,3 +461,86 @@ class TestAddressingSurvivesCoalescing:
)
assert client.queue.events[0].payload.addressed is True
+
+
+async def test_a_grouped_post_addressed_to_the_agent_is_reported_once() -> None:
+ """Three files in one post are one message asked of the agent, not three:
+ the report follows the coalesced payload, not the parts."""
+ client = _fake_client()
+ for index in range(3):
+ await AgentConsumer.on_media(
+ client,
+ _room(),
+ _media_event(
+ body="@agent-a look at these" if index == 0 else f"f{index}.png",
+ event_id=f"$part-{index}",
+ group={"id": "grp-r", "index": index, "total": 3},
+ ),
+ )
+
+ assert client._message_telemetry.addressed == [
+ {
+ "tenant_id": "tenant-1",
+ "room_id": "room-1",
+ "sender_transport_user_id": "@alice:s",
+ "from_platform": False,
+ "agent_metadata": None,
+ "agent_live": True,
+ "has_attachment": True,
+ }
+ ]
+
+
[email protected]("late_index", [0, 2])
+async def test_a_post_split_by_the_timeout_is_reported_once(late_index: int) -> None:
+ """A group that times out incomplete is delivered again when the rest
+ arrives. In a direct room both pieces address the agent, but it is one
+ post: counted with its first part, as the transport counts it."""
+ original = ac.ATTACHMENT_GROUP_TIMEOUT_SECONDS
+ ac.ATTACHMENT_GROUP_TIMEOUT_SECONDS = 0.01
+ try:
+ client = _fake_client()
+
+ async def _direct_room(_event: Any, _meta: RoomMeta) -> bool:
+ return True
+
+ client._addressed = _direct_room
+ on_time = [index for index in range(3) if index != late_index]
+ for batch in (on_time, [late_index]):
+ for index in batch:
+ await AgentConsumer.on_media(
+ client,
+ _room(),
+ _media_event(
+ body=f"f{index}.png",
+ event_id=f"$part-{index}",
+ group={"id": "grp-late", "index": index, "total": 3},
+ ),
+ )
+ await asyncio.sleep(0.15)
+ finally:
+ ac.ATTACHMENT_GROUP_TIMEOUT_SECONDS = original
+
+ assert len(client.queue.events) == 2
+ assert len(client._message_telemetry.addressed) == 1
+
+
+async def test_media_skips_the_availability_check_while_telemetry_is_off() -> None:
+ """The media path reads it only to report it, so off means no query."""
+ client = _fake_client()
+ client._message_telemetry.enabled = False
+ await AgentConsumer.on_media(
+ client, _room(), _media_event(body="@agent-a look", event_id="$solo")
+ )
+
+ assert client.availability_checks == 0
+ assert len(client.queue.events) == 1
+
+
+async def test_media_that_does_not_address_the_agent_is_not_reported() -> None:
+ client = _fake_client()
+ await AgentConsumer.on_media(
+ client, _room(), _media_event(body="just a picture", event_id="$solo")
+ )
+
+ assert client._message_telemetry.addressed == []
diff --git a/core/tests/switch_core/clients/test_agent_consumer_pool_checkouts.py b/core/tests/switch_core/clients/test_agent_consumer_pool_checkouts.py
index bc9a5cdad..b43fee430 100644
--- a/core/tests/switch_core/clients/test_agent_consumer_pool_checkouts.py
+++ b/core/tests/switch_core/clients/test_agent_consumer_pool_checkouts.py
@@ -142,6 +142,9 @@ def _client(
live_connection_ids=client._connections.live_connection_ids,
)
client._budget_guard = BudgetGuard(BudgetStore())
+ client._message_telemetry = SimpleNamespace( # type: ignore[assignment]
+ agent_addressed=lambda **_k: None
+ )
client._frontend_base_url = None
client._room_meta = {
TRANSPORT_ROOM_ID: RoomMeta(
diff --git a/core/tests/switch_core/clients/test_agent_consumer_reports_addressed.py b/core/tests/switch_core/clients/test_agent_consumer_reports_addressed.py
new file mode 100644
index 000000000..9b265ac44
--- /dev/null
+++ b/core/tests/switch_core/clients/test_agent_consumer_reports_addressed.py
@@ -0,0 +1,289 @@
+"""An agent's consumer reports each message its agent was actually asked to act on.
+
+"Asked" means addressed *and* let through: a message the addressing policy or a
+budget turned away was not a request the agent received, and Switch's own
+auto-replies are not anyone asking for anything. Whether the agent was there
+to take it is reported alongside, not used to leave the message out.
+"""
+
+from __future__ import annotations
+
+from contextlib import asynccontextmanager
+from types import SimpleNamespace
+from typing import Any
+
+import pytest
+
+from switch_core.clients import agent_consumer as agent_consumer_module
+from switch_core.clients.admin_messages import PLATFORM_MARKER
+from switch_core.clients.agent_consumer import (
+ AUTO_REPLY_FLAG,
+ AgentConsumer,
+ HostedNote,
+ _GateOutcome,
+)
+from switch_core.clients.room_meta import RoomMeta
+from switch_core.observability.metrics import MetricsRegistry, install, uninstall
+from switch_core.observability.throttle import WarningThrottle
+from switch_core.transport import InboundMessage, RoomRef
+
+
+@asynccontextmanager
+async def _session_factory(): # type: ignore[no-untyped-def]
+ yield object()
+
+
+class _FakeMessageTelemetry:
+ def __init__(self) -> None:
+ self.addressed: list[dict[str, Any]] = []
+
+ def agent_addressed(self, **kwargs: Any) -> None:
+ self.addressed.append(kwargs)
+
+
+class _BrokenMessageTelemetry:
+ def agent_addressed(self, **kwargs: Any) -> None:
+ raise RuntimeError("telemetry bug")
+
+
+def _consumer(
+ *,
+ addressed: bool,
+ allowed: bool = True,
+ live: bool = True,
+ hosted: HostedNote | None = None,
+) -> SimpleNamespace:
+ """An AgentConsumer stand-in with addressing, the policy gate, whether the
+ agent is live in the room and a hosted agent's mailbox fixed by the test."""
+ meta = RoomMeta(
+ room_id="room-1", name="Room", bridge_id=None, channel_type="channel_public"
+ )
+
+ async def _resolve_room_meta(_transport_room_id: str) -> RoomMeta:
+ return meta
+
+ async def _compute_addressed(_s: object, _e: object, _m: object) -> bool:
+ return addressed
+
+ async def _fresh_agent(_s: object) -> object:
+ return ns.agent
+
+ async def _gate_addressed(
+ _s: object, _a: object, _e: object, _m: object
+ ) -> _GateOutcome:
+ return _GateOutcome(addressed=allowed, refusal=None)
+
+ async def _is_available(_s: object, _a: object, _r: str) -> bool:
+ return live
+
+ async def _reply_when_unavailable_here(
+ _s: object, _a: object, _m: object, _h: str
+ ) -> str | None:
+ return None
+
+ async def _note_hosted_addressed(_a: object, _e: object) -> HostedNote | None:
+ return hosted
+
+ ns = SimpleNamespace(
+ agent=SimpleNamespace(
+ id="agent-1", name="agent-a", metadata_={"known_agent_type": "codex"}
+ ),
+ tenant_id="tenant-1",
+ session_factory=_session_factory,
+ _resolve_room_meta=_resolve_room_meta,
+ _addressed_without_lookup=lambda _e, _m: None,
+ _compute_addressed=_compute_addressed,
+ _fresh_agent=_fresh_agent,
+ _gate_addressed=_gate_addressed,
+ _is_available=_is_available,
+ _reply_when_unavailable_here=_reply_when_unavailable_here,
+ _note_hosted_addressed=_note_hosted_addressed,
+ _post_auto_reply=_ignore_post,
+ _sender_handle=lambda _e: "@alice",
+ _triggered_by_auto_reply=AgentConsumer._triggered_by_auto_reply,
+ _event_buffer=SimpleNamespace(enqueue=lambda *a, **k: ns.enqueued.append(a)),
+ enqueued=[],
+ _message_telemetry=_FakeMessageTelemetry(),
+ )
+ ns._report_addressed = AgentConsumer._report_addressed.__get__(ns)
+ return ns
+
+
+async def _ignore_post(*_args: object, **_kwargs: object) -> None:
+ return None
+
+
+def _message(**content: object) -> InboundMessage:
+ return InboundMessage(
+ room_id="!room:s",
+ event_id="$evt",
+ sender="@alice:s",
+ timestamp=0,
+ content={"sender_name": "alice", **content},
+ body="@agent-a can you help",
+ sender_name="alice",
+ )
+
+
+async def _deliver(consumer: SimpleNamespace, event: InboundMessage) -> None:
+ await AgentConsumer.on_message(consumer, RoomRef(room_id="!room:s"), event) # type: ignore[arg-type]
+
+
+async def test_an_addressed_message_is_reported_once() -> None:
+ consumer = _consumer(addressed=True)
+
+ await _deliver(consumer, _message())
+
+ assert consumer._message_telemetry.addressed == [
+ {
+ "tenant_id": "tenant-1",
+ "room_id": "room-1",
+ "sender_transport_user_id": "@alice:s",
+ "from_platform": False,
+ "agent_metadata": {"known_agent_type": "codex"},
+ "agent_live": True,
+ "has_attachment": False,
+ }
+ ]
+
+
+async def test_room_chatter_is_not_reported() -> None:
+ consumer = _consumer(addressed=False)
+
+ await _deliver(consumer, _message())
+
+ assert consumer._message_telemetry.addressed == []
+
+
+async def test_a_message_the_policy_turned_away_is_not_reported() -> None:
+ consumer = _consumer(addressed=True, allowed=False)
+
+ await _deliver(consumer, _message())
+
+ assert consumer._message_telemetry.addressed == []
+
+
+async def test_an_auto_reply_is_not_reported() -> None:
+ consumer = _consumer(addressed=True)
+
+ await _deliver(consumer, _message(**{AUTO_REPLY_FLAG: True}))
+
+ assert consumer._message_telemetry.addressed == []
+
+
+async def test_a_platform_message_says_it_came_from_the_platform() -> None:
+ consumer = _consumer(addressed=True)
+
+ await _deliver(consumer, _message(**{PLATFORM_MARKER: {}}))
+
+ [report] = consumer._message_telemetry.addressed
+ assert report["from_platform"] is True
+
+
+async def test_a_message_to_an_offline_agent_is_reported_as_not_live() -> None:
+ consumer = _consumer(addressed=True, live=False)
+
+ await _deliver(consumer, _message())
+
+ [report] = consumer._message_telemetry.addressed
+ assert report["agent_live"] is False
+
+
+async def test_a_message_a_hosted_agents_mailbox_refused_is_reported_as_not_live() -> (
+ None
+):
+ """A stopped or broken cloud worker is an agent that was asked and was not
+ there, the same as any other offline agent — not a message nobody sent."""
+ consumer = _consumer(
+ addressed=True,
+ live=False,
+ hosted=HostedNote(
+ launch=None, machine=None, refusal="worker stopped", deliver=False
+ ),
+ )
+
+ await _deliver(consumer, _message())
+
+ [report] = consumer._message_telemetry.addressed
+ assert report["agent_live"] is False
+
+
+async def test_a_message_a_live_hosted_worker_could_not_take_is_not_live() -> None:
+ """A full mailbox or a lost provider refuses the message even with a
+ worker attached, so the agent could not act on it."""
+ consumer = _consumer(
+ addressed=True,
+ live=True,
+ hosted=HostedNote(
+ launch=None, machine=None, refusal="mailbox full", deliver=False
+ ),
+ )
+
+ await _deliver(consumer, _message())
+
+ [report] = consumer._message_telemetry.addressed
+ assert report["agent_live"] is False
+
+
+async def test_a_message_held_for_a_waking_hosted_agent_is_reported() -> None:
+ consumer = _consumer(
+ addressed=True,
+ live=False,
+ hosted=HostedNote(launch=None, machine=None, refusal=None, deliver=True),
+ )
+
+ await _deliver(consumer, _message())
+
+ assert len(consumer._message_telemetry.addressed) == 1
+
+
+async def test_a_redelivery_the_mailbox_already_holds_is_not_counted_again() -> None:
+ """The same message seen a second time is one request, not two."""
+ consumer = _consumer(
+ addressed=True,
+ hosted=HostedNote(launch=None, machine=None, refusal=None, deliver=False),
+ )
+
+ await _deliver(consumer, _message())
+
+ assert consumer._message_telemetry.addressed == []
+
+
+async def test_a_telemetry_failure_does_not_cost_the_agent_the_message(
+ monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
+) -> None:
+ """Caught, so it is counted, and logged once rather than once per
+ message."""
+ consumer = _consumer(addressed=True)
+ consumer._message_telemetry = _BrokenMessageTelemetry()
+ registry = MetricsRegistry()
+ install(registry)
+ monkeypatch.setattr(
+ agent_consumer_module, "_report_failures", WarningThrottle(60.0)
+ )
+
+ try:
+ with caplog.at_level("ERROR"):
+ await _deliver(consumer, _message())
+ await _deliver(consumer, _message())
+ failures = next(
+ p
+ for p in registry.collect()
+ if p.name == "switch.messages.observer_failures"
+ )
+ finally:
+ uninstall()
+
+ assert len(consumer.enqueued) == 2
+ assert failures.numbers[0].value == 2.0
+ assert dict(failures.numbers[0].attributes) == {"actor": "agent"}
+ assert (
+ len(
+ [
+ r
+ for r in caplog.records
+ if "Could not report a message addressed" in r.getMessage()
+ ]
+ )
+ == 1
+ )
diff --git a/core/tests/switch_core/clients/test_client_tenant_plumbing.py b/core/tests/switch_core/clients/test_client_tenant_plumbing.py
index d58d88d0b..18e2eae70 100644
--- a/core/tests/switch_core/clients/test_client_tenant_plumbing.py
+++ b/core/tests/switch_core/clients/test_client_tenant_plumbing.py
@@ -51,6 +51,7 @@ from switch_core.db.stores.usage_store import UsageStore
from switch_core.tenant_context import tenant_scope
from switch_core.transport.ephemeral import EphemeralBus
from switch_core.transport.invites import InviteBus
+from switch_core.transport.observer import IgnoreParticipantMessages
def _factory(session_factory: async_sessionmaker[AsyncSession]) -> ClientFactory:
@@ -66,6 +67,7 @@ def _factory(session_factory: async_sessionmaker[AsyncSession]) -> ClientFactory
invites=InviteBus(),
ephemeral=EphemeralBus(),
room_cache=MagicMock(),
+ message_observer=IgnoreParticipantMessages(),
)
factory.register("user", Actor)
return factory
diff --git a/core/tests/switch_core/clients/test_no_session_reply_threading.py b/core/tests/switch_core/clients/test_no_session_reply_threading.py
index 63139edb6..dcbb6149b 100644
--- a/core/tests/switch_core/clients/test_no_session_reply_threading.py
+++ b/core/tests/switch_core/clients/test_no_session_reply_threading.py
@@ -99,6 +99,7 @@ def _fake_self(
_triggered_by_auto_reply=AgentConsumer._triggered_by_auto_reply,
actor=SimpleNamespace(send_message=send_message),
_event_buffer=SimpleNamespace(enqueue=lambda *a, **k: None),
+ _report_addressed=lambda *a, **k: None,
)
# Exercise the real sender-tagging and auto-reply helpers.
ns._sender_handle = AgentConsumer._sender_handle.__get__(ns)
diff --git a/core/tests/switch_core/gateway/test_hosted_mailbox.py b/core/tests/switch_core/gateway/test_hosted_mailbox.py
index 39ea47219..836eed41d 100644
--- a/core/tests/switch_core/gateway/test_hosted_mailbox.py
+++ b/core/tests/switch_core/gateway/test_hosted_mailbox.py
@@ -802,6 +802,7 @@ async def agent_client(app) -> SimpleNamespace:
)
posted: list[str] = []
enqueued: list[AgentEvent] = []
+ reported: list[dict[str, Any]] = []
async def send_message(_room_id: str, body: str, **_kwargs: Any) -> str:
posted.append(body)
@@ -847,13 +848,19 @@ async def agent_client(app) -> SimpleNamespace:
_waking_notice_revisions={},
_unreachable_notice_revisions={},
actor=SimpleNamespace(send_message=send_message),
+ _message_telemetry=SimpleNamespace(
+ enabled=True,
+ agent_addressed=lambda **kwargs: reported.append(kwargs),
+ ),
posted=posted,
enqueued=enqueued,
+ reported=reported,
)
for name in (
"_note_hosted_addressed",
"_emit_media",
"_post_auto_reply",
+ "_report_addressed",
"_sender_handle",
):
setattr(client, name, getattr(AgentConsumer, name).__get__(client))
@@ -893,6 +900,9 @@ async def test_media_to_an_errored_running_machine_is_refused_not_queued(
assert await rows(app) == {}
assert client.posted == [f"@someone {_HOSTED_MACHINE_ERROR_MESSAGE}"]
assert client.enqueued == []
+ # Still a message the agent was asked, by someone it could not answer.
+ [report] = client.reported
+ assert report["agent_live"] is False
async def test_task_delegate_to_an_errored_running_machine_is_refused(mailbox_app):
@@ -935,6 +945,30 @@ async def test_mention_to_an_errored_running_machine_is_answered_once(mailbox_ap
assert await rows(app) == {}
assert client.posted == [f"@someone {_HOSTED_MACHINE_ERROR_MESSAGE}"]
assert client.enqueued == []
+ [report] = client.reported
+ assert report["agent_live"] is False
+
+
+async def test_a_mention_the_mailbox_takes_is_a_message_the_agent_received(
+ mailbox_app,
+):
+ """Held for the worker rather than refused: asked, while not yet live."""
+ app = mailbox_app
+ client = await agent_client(app)
+ message = InboundMessage(
+ room_id="!room:example.com",
+ event_id="$m1",
+ sender="@someone:example.com",
+ timestamp=1700000000000,
+ content={"sender_name": "someone"},
+ body="@agent hello",
+ sender_name="someone",
+ )
+ await AgentConsumer.on_message(
+ client, RoomRef(room_id="!room:example.com"), message
+ ) # type: ignore[arg-type]
+ [report] = client.reported
+ assert report["agent_live"] is False
@pytest.mark.parametrize(
diff --git a/core/tests/switch_core/observability/test_otlp.py b/core/tests/switch_core/observability/test_otlp.py
index 86671a6da..45f8676e4 100644
--- a/core/tests/switch_core/observability/test_otlp.py
+++ b/core/tests/switch_core/observability/test_otlp.py
@@ -18,6 +18,7 @@ from switch_core.observability.otlp import (
MetricPayload,
NumberPoint,
OtlpClient,
+ OtlpPartialRejection,
OtlpResource,
OtlpSendError,
build_logs_payload,
@@ -292,6 +293,19 @@ async def test_partial_rejection_inside_a_200_is_raised():
await client.post("metrics", {})
+async def test_a_partial_rejection_says_how_many_records_and_why():
+ """A batching caller reports what it lost from this, not the whole batch."""
+ body = {"partialSuccess": {"rejectedLogRecords": "2", "errorMessage": "too big"}}
+ client = _client(lambda request: httpx.Response(200, json=body))
+
+ with pytest.raises(OtlpPartialRejection) as raised:
+ await client.post("logs", {})
+
+ assert raised.value.rejected == 2
+ assert raised.value.reason == "too big"
+
+
@pytest.mark.asyncio
async def test_empty_partial_success_is_not_a_failure():
client = _client(lambda request: httpx.Response(200, json={"partialSuccess": {}}))
diff --git a/core/tests/switch_core/observability/test_throttle.py b/core/tests/switch_core/observability/test_throttle.py
new file mode 100644
index 000000000..54e72b1b3
--- /dev/null
+++ b/core/tests/switch_core/observability/test_throttle.py
@@ -0,0 +1,55 @@
+"""The rate limit on warnings that could otherwise fire once per message."""
+
+from __future__ import annotations
+
+from types import SimpleNamespace
+
+import pytest
+
+from switch_core.observability import throttle
+from switch_core.observability.throttle import WarningThrottle
+
+
+class _Clock:
+ def __init__(self, now: float) -> None:
+ self.now = now
+
+ def __call__(self) -> float:
+ return self.now
+
+
+def clock(monkeypatch: pytest.MonkeyPatch) -> _Clock:
+ fake = _Clock(5.0)
+ monkeypatch.setattr(throttle, "time", SimpleNamespace(monotonic=fake))
+ return fake
+
+
+def test_the_first_occurrence_logs_even_just_after_boot(clock: _Clock) -> None:
+ """The monotonic clock counts from boot, so a "last logged at 0" start
+ would swallow every warning for the first minute of a fresh host."""
+ assert WarningThrottle(60.0).note() == 1
+
+
+def test_occurrences_inside_the_interval_are_counted_not_logged(
+ clock: _Clock,
+) -> None:
+ warnings = WarningThrottle(60.0)
+ warnings.note()
+
+ clock.now += 30
+ assert warnings.note() is None
+ assert warnings.note() is None
+
+ clock.now += 31
+ assert warnings.note() == 3
+
+
+def test_what_was_never_logged_can_be_taken_at_the_end(clock: _Clock) -> None:
+ warnings = WarningThrottle(60.0)
+ warnings.note()
+ warnings.note()
+ warnings.note()
+
+ assert warnings.take_pending() == 2
+ assert warnings.take_pending() == 0
diff --git a/core/tests/switch_core/session_activity/test_session_started_route.py b/core/tests/switch_core/session_activity/test_session_started_route.py
index ff55dfbd1..b097ba00f 100644
--- a/core/tests/switch_core/session_activity/test_session_started_route.py
+++ b/core/tests/switch_core/session_activity/test_session_started_route.py
@@ -40,7 +40,7 @@ class _RecordingSink:
def __init__(self) -> None:
self.sent: list[TelemetryRecord] = []
- async def send(self, record: TelemetryRecord) -> None:
+ def send(self, record: TelemetryRecord) -> None:
self.sent.append(record)
async def aclose(self) -> None:
diff --git a/core/tests/switch_core/telemetry/test_bridge_connect_reporting.py b/core/tests/switch_core/telemetry/test_bridge_connect_reporting.py
index 9003eeef1..09324f3fe 100644
--- a/core/tests/switch_core/telemetry/test_bridge_connect_reporting.py
+++ b/core/tests/switch_core/telemetry/test_bridge_connect_reporting.py
@@ -57,7 +57,7 @@ class _RecordingSink:
def __init__(self) -> None:
self.sent: list[TelemetryRecord] = []
- async def send(self, record: TelemetryRecord) -> None:
+ def send(self, record: TelemetryRecord) -> None:
self.sent.append(record)
async def aclose(self) -> None:
diff --git a/core/tests/switch_core/telemetry/test_catalogue.py b/core/tests/switch_core/telemetry/test_catalogue.py
index 15a1354de..950f19b4a 100644
--- a/core/tests/switch_core/telemetry/test_catalogue.py
+++ b/core/tests/switch_core/telemetry/test_catalogue.py
@@ -177,3 +177,23 @@ class TestNaming:
keeps `room_created` here apart from the Console's."""
assert wire_name("room_created") == f"{EVENT_NAME_PREFIX}.room_created"
assert EVENT_NAME_PREFIX == "switch_core"
+
+
+class TestAgentRuntimes:
+ """`unknown` is a lookup that failed, so only the event that looks the
+ agent up may carry it. Everywhere else the runtime comes from an agent in
+ hand, and `unknown` there would be a bug hiding as a value."""
+
+ def test_the_sent_event_may_say_its_runtime_is_unknown(self) -> None:
+ payload = _valid_payload("agent_message_sent")
+ payload["known_agent_type"] = "unknown"
+ validate("agent_message_sent", payload) # type: ignore[arg-type]
+
+ @pytest.mark.parametrize(
+ "event", ["agent_message_received", "first_agent_registered"]
+ )
+ def test_events_with_the_agent_in_hand_may_not(self, event: str) -> None:
+ payload = _valid_payload(event)
+ payload["known_agent_type"] = "unknown"
+ with pytest.raises(TelemetryCatalogueError, match="expected one of"):
+ validate(event, payload) # type: ignore[arg-type]
diff --git a/core/tests/switch_core/telemetry/test_deployment_and_reporter.py b/core/tests/switch_core/telemetry/test_deployment_and_reporter.py
index e85d01689..2acfd7639 100644
--- a/core/tests/switch_core/telemetry/test_deployment_and_reporter.py
+++ b/core/tests/switch_core/telemetry/test_deployment_and_reporter.py
@@ -56,7 +56,7 @@ class _RecordingSink:
def __init__(self) -> None:
self.sent: list[TelemetryRecord] = []
- async def send(self, record: TelemetryRecord) -> None:
+ def send(self, record: TelemetryRecord) -> None:
self.sent.append(record)
async def aclose(self) -> None:
diff --git a/core/tests/switch_core/telemetry/test_first_room_active.py b/core/tests/switch_core/telemetry/test_first_room_active.py
index 5fa1af07c..9b5690512 100644
--- a/core/tests/switch_core/telemetry/test_first_room_active.py
+++ b/core/tests/switch_core/telemetry/test_first_room_active.py
@@ -30,7 +30,7 @@ class _RecordingSink:
def __init__(self) -> None:
self.sent: list[TelemetryRecord] = []
- async def send(self, record: TelemetryRecord) -> None:
+ def send(self, record: TelemetryRecord) -> None:
self.sent.append(record)
async def aclose(self) -> None:
diff --git a/core/tests/switch_core/telemetry/test_merge_fixes.py b/core/tests/switch_core/telemetry/test_merge_fixes.py
index 40295e17e..7264f77af 100644
--- a/core/tests/switch_core/telemetry/test_merge_fixes.py
+++ b/core/tests/switch_core/telemetry/test_merge_fixes.py
@@ -36,7 +36,7 @@ class _RecordingSink:
def __init__(self) -> None:
self.sent: list[TelemetryRecord] = []
- async def send(self, record: TelemetryRecord) -> None:
+ def send(self, record: TelemetryRecord) -> None:
self.sent.append(record)
async def aclose(self) -> None:
diff --git a/core/tests/switch_core/telemetry/test_message_events.py b/core/tests/switch_core/telemetry/test_message_events.py
new file mode 100644
index 000000000..d188da64b
--- /dev/null
+++ b/core/tests/switch_core/telemetry/test_message_events.py
@@ -0,0 +1,1050 @@
+"""The per-message events, against real Postgres.
+
+What these pin is what each event says about the room and the sender — the
+platform, the room's size, who spoke — because those are read from the
+database by the worker rather than handed over by the sender, and a wrong join
+there would be a wrong chart with no error anywhere.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import time
+import uuid
+from types import SimpleNamespace
+
+import pytest
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
+
+from switch_core.db.models import (
+ Agent,
+ ApiKey,
+ Client,
+ ClientRoom,
+ CollaborationBridge,
+ Room,
+ User,
+)
+from switch_core.telemetry import messages as messages_module
+from switch_core.telemetry.messages import MessageTelemetry, _RoomFacts, _TtlCache
+from switch_core.telemetry.service import TelemetryService
+from switch_core.telemetry.sink import TelemetryRecord
+from switch_core.transport.observer import ParticipantMessage
+
+TENANT_ZERO = "00000000-0000-0000-0000-000000000000"
+
+
+class _RecordingSink:
+ def __init__(self) -> None:
+ self.sent: list[TelemetryRecord] = []
+
+ def send(self, record: TelemetryRecord) -> None:
+ self.sent.append(record)
+
+ async def aclose(self) -> None:
+ return None
+
+
+def _service(sink: _RecordingSink, *, enabled: bool = True) -> TelemetryService:
+ return TelemetryService(
+ sink=sink,
+ enabled=enabled,
+ client_id="deployment-uuid",
+ service_name="switch-core",
+ version="1.0.0",
+ environment=None,
+ telemetry_environment="prod",
+ telemetry_internal=False,
+ )
+
+
+async def _client(session: AsyncSession, client_type: str) -> Client:
+ client = Client(
+ transport_user_id=f"@{client_type}-{uuid.uuid4().hex[:8]}:test",
+ display_name=f"{client_type} client",
+ type=client_type,
+ )
+ session.add(client)
+ await session.flush()
+ return client
+
+
+async def _agent(session: AsyncSession, runtime: str) -> Agent:
+ slug = uuid.uuid4().hex[:10]
+ owner = User(
+ name="owner", email=f"owner-{slug}@test", role="user", password_hash="x"
+ )
+ session.add(owner)
+ await session.flush()
+ key = ApiKey(
+ user_id=owner.id,
+ key_hash=f"hash-{slug}",
+ encrypted_key="enc",
+ label="test",
+ type="agent",
+ )
+ backing = await _client(session, "agent")
+ session.add(key)
+ await session.flush()
+ agent = Agent(
+ name=f"agent-{slug[:6]}",
+ description="d",
+ agent_type="session_addressable",
+ connector_type="http",
+ integration_profile={},
+ client_id=backing.id,
+ api_key_id=key.id,
+ metadata_={"known_agent_type": runtime},
+ )
+ session.add(agent)
+ await session.flush()
+ return agent
+
+
+async def _room(
+ session: AsyncSession, *, bridge_platform: str | None, channel_type: str
+) -> Room:
+ bridge_id: str | None = None
+ if bridge_platform is not None:
+ bridge = CollaborationBridge(
+ type=bridge_platform,
+ display_name="workspace",
+ client_id=(await _client(session, "bridge")).id,
+ status="active",
+ )
+ session.add(bridge)
+ await session.flush()
+ bridge_id = bridge.id
+ room = Room(
+ transport_room_id=f"!{uuid.uuid4().hex[:10]}:test",
+ name="a room",
+ description="",
+ channel_type=channel_type,
+ bridge_id=bridge_id,
+ )
+ session.add(room)
+ await session.flush()
+ return room
+
+
+async def _join(session: AsyncSession, client_id: str, room: Room) -> None:
+ session.add(ClientRoom(client_id=client_id, room_id=room.id))
+ await session.flush()
+
+
+class _World:
+ """A Slack-bridged private room with two people, one Codex agent, and the
+ bridge's own member — which is neither, and must count as neither."""
+
+ room: Room
+ humans: list[Client]
+ agent: Agent
+
+
+async def _world(session_factory: async_sessionmaker[AsyncSession]) -> _World:
+ world = _World()
+ async with session_factory() as session:
+ world.room = await _room(
+ session, bridge_platform="slack", channel_type="channel_private"
+ )
+ world.humans = [await _client(session, "user") for _ in range(2)]
+ world.agent = await _agent(session, "codex")
+ for human in world.humans:
+ await _join(session, human.id, world.room)
+ await _join(session, world.agent.client_id, world.room)
+ await _join(session, (await _client(session, "bridge")).id, world.room)
+ await session.commit()
+ return world
+
+
+def _said(world: _World, sender_client_id: str, role: str) -> ParticipantMessage:
+ return ParticipantMessage(
+ tenant_id=TENANT_ZERO,
+ room_id=world.room.id,
+ sender_client_id=sender_client_id,
+ sender_role=role,
+ has_attachment=False,
+ in_thread=False,
+ )
+
+
+def _said_in(
+ room_id: str, *, sender_role: str = "human", sender_client_id: str = "someone"
+) -> ParticipantMessage:
+ return ParticipantMessage(
+ tenant_id=TENANT_ZERO,
+ room_id=room_id,
+ sender_client_id=sender_client_id,
+ sender_role=sender_role,
+ has_attachment=False,
+ in_thread=False,
+ )
+
+
+async def _a_slack_room(tenant_id: str, room_id: str) -> _RoomFacts:
+ return _RoomFacts(
+ bridge_platform="slack",
+ channel_type="channel_public",
+ user_count=1,
+ agent_count=1,
+ )
+
+
+class _Unreachable:
+ """A session factory for a database that will not answer."""
+
+ def __init__(self) -> None:
+ self.attempts = 0
+
+ def __call__(self) -> object:
+ self.attempts += 1
+ raise ConnectionError("database unreachable")
+
+
+async def _reported(
+ messages: MessageTelemetry, service: TelemetryService, sink: _RecordingSink
+) -> list[tuple[str, dict[str, object]]]:
+ await messages.aclose()
+ await service.aclose()
+ return [(record.name, record.properties) for record in sink.sent]
+
+
+class TestRoomMessageSent:
+ async def test_a_person_speaking_is_reported_with_the_room_it_was_said_in(
+ self, session_factory: async_sessionmaker[AsyncSession]
+ ) -> None:
+ world = await _world(session_factory)
+ sink = _RecordingSink()
+ service = _service(sink)
+ messages = MessageTelemetry(telemetry=service, session_factory=session_factory)
+
+ messages.observe(_said(world, world.humans[0].id, "human"))
+
+ assert await _reported(messages, service, sink) == [
+ (
+ "switch_core.room_message_sent",
+ {
+ "sender_kind": "user",
+ "bridge_platform": "slack",
+ "channel_type": "channel_private",
+ "room_user_count": 2,
+ "room_agent_count": 1,
+ "has_attachment": False,
+ "in_thread": False,
+ },
+ )
+ ]
+
+ async def test_an_internal_room_has_no_platform(
+ self, session_factory: async_sessionmaker[AsyncSession]
+ ) -> None:
+ async with session_factory() as session:
+ room = await _room(session, bridge_platform=None, channel_type="direct")
+ human = await _client(session, "user")
+ await _join(session, human.id, room)
+ await session.commit()
+ sink = _RecordingSink()
+ service = _service(sink)
+ messages = MessageTelemetry(telemetry=service, session_factory=session_factory)
+
+ messages.observe(
+ ParticipantMessage(
+ tenant_id=TENANT_ZERO,
+ room_id=room.id,
+ sender_client_id=human.id,
+ sender_role="human",
+ has_attachment=True,
+ in_thread=True,
+ )
+ )
+
+ [(_, properties)] = await _reported(messages, service, sink)
+ assert properties["bridge_platform"] == "none"
+ assert properties["channel_type"] == "direct"
+ assert properties["has_attachment"] is True
+ assert properties["in_thread"] is True
+
+ async def test_a_room_that_cannot_be_found_is_reported_as_unknown(
+ self, session_factory: async_sessionmaker[AsyncSession]
+ ) -> None:
+ # Still counted: a lookup problem must read as `unknown` in the chart,
+ # not as fewer messages.
+ sink = _RecordingSink()
+ service = _service(sink)
+ messages = MessageTelemetry(telemetry=service, session_factory=session_factory)
+
+ messages.observe(
+ ParticipantMessage(
+ tenant_id=TENANT_ZERO,
+ room_id=str(uuid.uuid4()),
+ sender_client_id="whoever",
+ sender_role="human",
+ has_attachment=False,
+ in_thread=False,
+ )
+ )
+
+ [(_, properties)] = await _reported(messages, service, sink)
+ assert properties["bridge_platform"] == "unknown"
+ assert properties["channel_type"] == "unknown"
+ assert properties["room_user_count"] == -1
+ assert properties["room_agent_count"] == -1
+
+
+class TestAgentMessageSent:
+ async def test_an_agent_speaking_is_also_reported_with_its_runtime(
+ self, session_factory: async_sessionmaker[AsyncSession]
+ ) -> None:
+ world = await _world(session_factory)
+ sink = _RecordingSink()
+ service = _service(sink)
+ messages = MessageTelemetry(telemetry=service, session_factory=session_factory)
+
+ messages.observe(_said(world, world.agent.client_id, "agent"))
+
+ reported = await _reported(messages, service, sink)
+ assert [name for name, _ in reported] == [
+ "switch_core.room_message_sent",
+ "switch_core.agent_message_sent",
+ ]
+ assert reported[0][1]["sender_kind"] == "agent"
+ assert reported[1][1] == {
+ "known_agent_type": "codex",
+ "bridge_platform": "slack",
+ "channel_type": "channel_private",
+ "room_user_count": 2,
+ "has_attachment": False,
+ "in_thread": False,
+ }
+
+ async def test_an_agent_that_cannot_be_found_has_an_unknown_runtime(
+ self, session_factory: async_sessionmaker[AsyncSession]
+ ) -> None:
+ """Not `none`, which is a real answer — an agent that declares no
+ runtime. A lookup problem must not read as a population of those."""
+ world = await _world(session_factory)
+ async with session_factory() as session:
+ unregistered = await _client(session, "agent")
+ await session.commit()
+ sink = _RecordingSink()
+ service = _service(sink)
+ messages = MessageTelemetry(telemetry=service, session_factory=session_factory)
+
+ messages.observe(_said(world, unregistered.id, "agent"))
+
+ reported = await _reported(messages, service, sink)
+ assert reported[1][1]["known_agent_type"] == "unknown"
+
+
+class TestAgentMessageReceived:
+ async def test_a_person_asking_an_agent_is_reported(
+ self, session_factory: async_sessionmaker[AsyncSession]
+ ) -> None:
+ world = await _world(session_factory)
+ sink = _RecordingSink()
+ service = _service(sink)
+ messages = MessageTelemetry(telemetry=service, session_factory=session_factory)
+
+ messages.agent_addressed(
+ tenant_id=TENANT_ZERO,
+ room_id=world.room.id,
+ sender_transport_user_id=world.humans[0].transport_user_id,
+ from_platform=False,
+ agent_metadata={"known_agent_type": "claude-code"},
+ agent_live=True,
+ has_attachment=False,
+ )
+
+ assert await _reported(messages, service, sink) == [
+ (
+ "switch_core.agent_message_received",
+ {
+ "sender_kind": "user",
+ "known_agent_type": "claude-code",
+ "bridge_platform": "slack",
+ "channel_type": "channel_private",
+ "has_attachment": False,
+ "agent_live": True,
+ },
+ )
+ ]
+
+ async def test_who_asked_is_told_apart(
+ self, session_factory: async_sessionmaker[AsyncSession]
+ ) -> None:
+ world = await _world(session_factory)
+ sink = _RecordingSink()
+ service = _service(sink)
+ messages = MessageTelemetry(telemetry=service, session_factory=session_factory)
+ async with session_factory() as session:
+ sender_agent = await _client(session, "agent")
+ await session.commit()
+
+ for sender, from_platform in (
+ (sender_agent.transport_user_id, False),
+ ("@admin:test", True),
+ ("@nobody-we-know:test", False),
+ ):
+ messages.agent_addressed(
+ tenant_id=TENANT_ZERO,
+ room_id=world.room.id,
+ sender_transport_user_id=sender,
+ from_platform=from_platform,
+ agent_metadata=None,
+ agent_live=False,
+ has_attachment=False,
+ )
+
+ reported = await _reported(messages, service, sink)
+ assert [p["sender_kind"] for _, p in reported] == [
+ "agent",
+ "platform",
+ "unknown",
+ ]
+ assert {p["known_agent_type"] for _, p in reported} == {"none"}
+
+
+class TestFailingLookups:
+ async def test_a_room_whose_lookup_failed_is_not_retried_and_warns_once(
+ self,
+ session_factory: async_sessionmaker[AsyncSession],
+ caplog: pytest.LogCaptureFixture,
+ ) -> None:
+ """A busy room whose lookup fails must not cost a query, and a
+ traceback, for every message said in it."""
+ world = await _world(session_factory)
+ sink = _RecordingSink()
+ service = _service(sink)
+ messages = MessageTelemetry(telemetry=service, session_factory=session_factory)
+ attempts = 0
+
+ async def _broken(tenant_id: str, room_id: str) -> None:
+ nonlocal attempts
+ attempts += 1
+ raise ConnectionError("database unreachable")
+
+ messages._load_room_facts = _broken # type: ignore[method-assign]
+
+ with caplog.at_level("WARNING"):
+ for _ in range(3):
+ messages.observe(_said(world, world.humans[0].id, "human"))
+ reported = await _reported(messages, service, sink)
+
+ assert attempts == 1
+ assert [p["bridge_platform"] for _, p in reported] == ["unknown"] * 3
+ assert (
+ len([r for r in caplog.records if "could not look up" in r.getMessage()])
+ == 1
+ )
+
+ async def test_one_failing_room_does_not_blank_the_others(self) -> None:
+ """The pause is the failing key's: one tenant or room that keeps
+ failing must not turn every other room's events into `unknown`."""
+ sink = _RecordingSink()
+ service = _service(sink)
+ messages = MessageTelemetry(
+ telemetry=service,
+ session_factory=None, # type: ignore[arg-type]
+ )
+ attempts: list[tuple[str, str]] = []
+
+ async def _bad_room_fails(tenant_id: str, room_id: str) -> _RoomFacts:
+ attempts.append((tenant_id, room_id))
+ if room_id == "bad":
+ raise ConnectionError("this room's lookup fails")
+ return await _a_slack_room(tenant_id, room_id)
+
+ messages._load_room_facts = _bad_room_fails # type: ignore[method-assign]
+ other_tenant = str(uuid.uuid4())
+
+ messages.observe(_said_in("bad"))
+ messages.observe(_said_in("good"))
+ messages.observe(
+ ParticipantMessage(
+ tenant_id=other_tenant,
+ room_id="elsewhere",
+ sender_client_id="someone",
+ sender_role="human",
+ has_attachment=False,
+ in_thread=False,
+ )
+ )
+ messages.observe(_said_in("bad"))
+ reported = await _reported(messages, service, sink)
+
+ assert [p["bridge_platform"] for _, p in reported] == [
+ "unknown",
+ "slack",
+ "slack",
+ "unknown",
+ ]
+ assert attempts == [
+ (TENANT_ZERO, "bad"),
+ (TENANT_ZERO, "good"),
+ (other_tenant, "elsewhere"),
+ ]
+
+ async def test_failures_in_a_row_pause_every_lookup(
+ self, caplog: pytest.LogCaptureFixture
+ ) -> None:
+ """Failing for several different keys with nothing succeeding between
+ is the database, not a bad row: then every lookup waits."""
+ sink = _RecordingSink()
+ service = _service(sink)
+ messages = MessageTelemetry(
+ telemetry=service,
+ session_factory=None, # type: ignore[arg-type]
+ )
+ attempts: list[str] = []
+
+ async def _broken(tenant_id: str, room_id: str) -> None:
+ attempts.append(room_id)
+ raise ConnectionError("database unreachable")
+
+ messages._load_room_facts = _broken # type: ignore[method-assign]
+
+ with caplog.at_level("WARNING"):
+ for room in ("a", "b", "c", "d", "e"):
+ messages.observe(_said_in(room))
+ reported = await _reported(messages, service, sink)
+
+ assert attempts == ["a", "b", "c"]
+ assert [p["bridge_platform"] for _, p in reported] == ["unknown"] * 5
+ assert "failed 3 times in a row, so every lookup pauses" in caplog.text
+
+ async def test_a_lookup_that_succeeds_resets_the_count(self) -> None:
+ sink = _RecordingSink()
+ service = _service(sink)
+ messages = MessageTelemetry(
+ telemetry=service,
+ session_factory=None, # type: ignore[arg-type]
+ )
+ attempts: list[str] = []
+
+ async def _only_good_answers(tenant_id: str, room_id: str) -> _RoomFacts:
+ attempts.append(room_id)
+ if room_id != "good":
+ raise ConnectionError("this room's lookup fails")
+ return await _a_slack_room(tenant_id, room_id)
+
+ messages._load_room_facts = _only_good_answers # type: ignore[method-assign]
+
+ for room in ("a", "b", "good", "c", "d"):
+ messages.observe(_said_in(room))
+ await _reported(messages, service, sink)
+
+ assert attempts == ["a", "b", "good", "c", "d"]
+
+
+class TestCaching:
+ async def test_a_senders_kind_is_read_once_while_cached(
+ self, session_factory: async_sessionmaker[AsyncSession]
+ ) -> None:
+ world = await _world(session_factory)
+ sink = _RecordingSink()
+ service = _service(sink)
+ messages = MessageTelemetry(telemetry=service, session_factory=session_factory)
+
+ def _ask() -> None:
+ messages.agent_addressed(
+ tenant_id=TENANT_ZERO,
+ room_id=world.room.id,
+ sender_transport_user_id=world.humans[0].transport_user_id,
+ from_platform=False,
+ agent_metadata=None,
+ agent_live=True,
+ has_attachment=False,
+ )
+
+ _ask()
+ await messages._queue.join()
+ async with session_factory() as session:
+ sender = await session.get(Client, world.humans[0].id)
+ assert sender is not None
+ sender.type = "agent"
+ await session.commit()
+ _ask()
+
+ reported = await _reported(messages, service, sink)
+ assert [p["sender_kind"] for _, p in reported] == ["user", "user"]
+
+ async def test_an_agents_runtime_is_read_once_while_cached(
+ self, session_factory: async_sessionmaker[AsyncSession]
+ ) -> None:
+ world = await _world(session_factory)
+ sink = _RecordingSink()
+ service = _service(sink)
+ messages = MessageTelemetry(telemetry=service, session_factory=session_factory)
+
+ messages.observe(_said(world, world.agent.client_id, "agent"))
+ await messages._queue.join()
+ async with session_factory() as session:
+ agent = await session.get(Agent, world.agent.id)
+ assert agent is not None
+ agent.metadata_ = {"known_agent_type": "claude-code"}
+ await session.commit()
+ messages.observe(_said(world, world.agent.client_id, "agent"))
+
+ reported = await _reported(messages, service, sink)
+ assert [
+ p["known_agent_type"]
+ for name, p in reported
+ if name == "switch_core.agent_message_sent"
+ ] == ["codex", "codex"]
+
+ async def test_a_room_that_cannot_be_found_is_looked_up_once(self) -> None:
+ """A room deleted while its last messages were queued must not cost a
+ query per message."""
+ sink = _RecordingSink()
+ service = _service(sink)
+ messages = MessageTelemetry(
+ telemetry=service,
+ session_factory=None, # type: ignore[arg-type]
+ )
+ loads = 0
+
+ async def _gone(tenant_id: str, room_id: str) -> None:
+ nonlocal loads
+ loads += 1
+ return None
+
+ messages._load_room_facts = _gone # type: ignore[method-assign]
+
+ for _ in range(3):
+ messages.observe(_said_in("deleted-room"))
+ reported = await _reported(messages, service, sink)
+
+ assert loads == 1
+ assert [p["bridge_platform"] for _, p in reported] == ["unknown"] * 3
+
+ async def test_a_sender_or_agent_that_cannot_be_found_is_looked_up_once(
+ self, session_factory: async_sessionmaker[AsyncSession]
+ ) -> None:
+ """The miss is cached like a hit: a row that appears afterwards is not
+ seen until the entry expires."""
+ world = await _world(session_factory)
+ sink = _RecordingSink()
+ service = _service(sink)
+ messages = MessageTelemetry(telemetry=service, session_factory=session_factory)
+ late_sender = f"@late-{uuid.uuid4().hex[:8]}:test"
+ async with session_factory() as session:
+ unregistered = await _client(session, "agent")
+ await session.commit()
+
+ def _activity() -> None:
+ messages.agent_addressed(
+ tenant_id=TENANT_ZERO,
+ room_id=world.room.id,
+ sender_transport_user_id=late_sender,
+ from_platform=False,
+ agent_metadata=None,
+ agent_live=True,
+ has_attachment=False,
+ )
+ messages.observe(_said(world, unregistered.id, "agent"))
+
+ _activity()
+ await messages._queue.join()
+ async with session_factory() as session:
+ session.add(
+ Client(transport_user_id=late_sender, display_name="late", type="user")
+ )
+ owner = await _agent(session, "codex")
+ owner.client_id = unregistered.id
+ await session.commit()
+ _activity()
+
+ reported = await _reported(messages, service, sink)
+ assert [
+ p["sender_kind"]
+ for name, p in reported
+ if name == "switch_core.agent_message_received"
+ ] == ["unknown", "unknown"]
+ assert [
+ p["known_agent_type"]
+ for name, p in reported
+ if name == "switch_core.agent_message_sent"
+ ] == ["unknown", "unknown"]
+
+
+class TestTtlCache:
+ def test_an_entry_expires_after_its_age(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ clock = SimpleNamespace(now=100.0)
+ monkeypatch.setattr(
+ messages_module, "time", SimpleNamespace(monotonic=lambda: clock.now)
+ )
+ cache: _TtlCache[str, int] = _TtlCache(ttl_seconds=10, max_entries=5)
+ cache.put("a", 1)
+
+ clock.now += 9
+ assert cache.get("a") == 1
+ clock.now += 2
+ assert cache.get("a") is None
+
+ def test_the_least_recently_used_entry_goes_first(self) -> None:
+ cache: _TtlCache[str, int] = _TtlCache(ttl_seconds=60, max_entries=2)
+ cache.put("a", 1)
+ cache.put("b", 2)
+ cache.get("a")
+ cache.put("c", 3)
+
+ assert cache.get("b") is None
+ assert cache.get("a") == 1
+ assert cache.get("c") == 3
+
+
+class TestBackpressure:
+ async def test_a_full_queue_drops_with_a_warning_and_a_final_count(
+ self, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
+ ) -> None:
+ """Bounded rather than growing inside the process it measures, and the
+ loss is said out loud — including drops after the last warning."""
+ monkeypatch.setattr(messages_module, "_QUEUE_SIZE", 1)
+ sink = _RecordingSink()
+ service = _service(sink)
+ messages = MessageTelemetry(
+ telemetry=service,
+ session_factory=None, # type: ignore[arg-type]
+ )
+ messages._load_room_facts = _a_slack_room # type: ignore[method-assign]
+
+ with caplog.at_level("WARNING"):
+ for _ in range(4):
+ messages.observe(_said_in("room"))
+ reported = await _reported(messages, service, sink)
+
+ behind = [
+ r.getMessage()
+ for r in caplog.records
+ if "Message telemetry is behind" in r.getMessage()
+ ]
+ assert len(reported) == 1
+ assert len(behind) == 2
+ assert "1 event(s) dropped" in behind[0]
+ assert "2 event(s) dropped" in behind[1]
+
+
+class TestFailingSenderAndRuntimeLookups:
+ async def test_a_sender_lookup_that_fails_reports_unknown_and_pauses(
+ self, caplog: pytest.LogCaptureFixture
+ ) -> None:
+ database = _Unreachable()
+ sink = _RecordingSink()
+ service = _service(sink)
+ messages = MessageTelemetry(
+ telemetry=service,
+ session_factory=database, # type: ignore[arg-type]
+ )
+ messages._load_room_facts = _a_slack_room # type: ignore[method-assign]
+
+ for sender in ("@alice:test", "@alice:test"):
+ messages.agent_addressed(
+ tenant_id=TENANT_ZERO,
+ room_id="room",
+ sender_transport_user_id=sender,
+ from_platform=False,
+ agent_metadata=None,
+ agent_live=True,
+ has_attachment=False,
+ )
+ with caplog.at_level("WARNING"):
+ reported = await _reported(messages, service, sink)
+
+ assert [p["sender_kind"] for _, p in reported] == ["unknown", "unknown"]
+ assert database.attempts == 1
+ assert "could not look up a message sender" in caplog.text
+
+ async def test_a_runtime_lookup_that_fails_reports_unknown_and_pauses(
+ self, caplog: pytest.LogCaptureFixture
+ ) -> None:
+ database = _Unreachable()
+ sink = _RecordingSink()
+ service = _service(sink)
+ messages = MessageTelemetry(
+ telemetry=service,
+ session_factory=database, # type: ignore[arg-type]
+ )
+ messages._load_room_facts = _a_slack_room # type: ignore[method-assign]
+
+ for _ in range(2):
+ messages.observe(
+ _said_in("room", sender_role="agent", sender_client_id="agent-client")
+ )
+ with caplog.at_level("WARNING"):
+ reported = await _reported(messages, service, sink)
+
+ assert [
+ p["known_agent_type"]
+ for name, p in reported
+ if name == "switch_core.agent_message_sent"
+ ] == ["unknown", "unknown"]
+ assert database.attempts == 1
+ assert "could not look up an agent's runtime" in caplog.text
+
+ async def test_failures_past_the_pause_still_warn_once_a_minute(
+ self, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
+ ) -> None:
+ monkeypatch.setattr(messages_module, "_LOOKUP_BACKOFF_SECONDS", 0.0)
+ sink = _RecordingSink()
+ service = _service(sink)
+ messages = MessageTelemetry(
+ telemetry=service,
+ session_factory=None, # type: ignore[arg-type]
+ )
+ attempts = 0
+
+ async def _broken(tenant_id: str, room_id: str) -> None:
+ nonlocal attempts
+ attempts += 1
+ raise ConnectionError("database unreachable")
+
+ messages._load_room_facts = _broken # type: ignore[method-assign]
+
+ with caplog.at_level("WARNING"):
+ for _ in range(3):
+ messages.observe(_said_in("room"))
+ await _reported(messages, service, sink)
+
+ assert attempts == 3
+ assert (
+ len([r for r in caplog.records if "could not look up" in r.getMessage()])
+ == 1
+ )
+
+ async def test_rooms_that_cannot_be_found_warn_once(
+ self, caplog: pytest.LogCaptureFixture
+ ) -> None:
+ sink = _RecordingSink()
+ service = _service(sink)
+ messages = MessageTelemetry(
+ telemetry=service,
+ session_factory=None, # type: ignore[arg-type]
+ )
+
+ async def _gone(tenant_id: str, room_id: str) -> None:
+ return None
+
+ messages._load_room_facts = _gone # type: ignore[method-assign]
+
+ with caplog.at_level("WARNING"):
+ messages.observe(_said_in("room-a"))
+ messages.observe(_said_in("room-b"))
+ reported = await _reported(messages, service, sink)
+
+ assert [p["bridge_platform"] for _, p in reported] == ["unknown", "unknown"]
+ assert (
+ len([r for r in caplog.records if "found no room" in r.getMessage()]) == 1
+ )
+
+
+class TestFailingReports:
+ async def test_an_event_that_cannot_be_reported_logs_once_not_per_message(
+ self,
+ session_factory: async_sessionmaker[AsyncSession],
+ caplog: pytest.LogCaptureFixture,
+ ) -> None:
+ """A bug that breaks every event would otherwise write a traceback per
+ message."""
+ world = await _world(session_factory)
+ sink = _RecordingSink()
+ service = _service(sink)
+ messages = MessageTelemetry(telemetry=service, session_factory=session_factory)
+
+ async def _broken(item: object) -> None:
+ raise RuntimeError("bug in building the event")
+
+ messages._report = _broken # type: ignore[method-assign]
+
+ with caplog.at_level("WARNING"):
+ for _ in range(3):
+ messages.observe(_said(world, world.humans[0].id, "human"))
+ await _reported(messages, service, sink)
+
+ errors = [
+ r
+ for r in caplog.records
+ if "Could not report a message event" in r.getMessage()
+ ]
+ assert len(errors) == 1
+ assert errors[0].exc_info is not None
+ # The two after it are counted at shutdown rather than lost.
+ assert "2 more message event(s) could not be reported" in caplog.text
+
+ async def test_a_sender_that_cannot_be_found_is_disclosed(
+ self,
+ session_factory: async_sessionmaker[AsyncSession],
+ caplog: pytest.LogCaptureFixture,
+ ) -> None:
+ world = await _world(session_factory)
+ sink = _RecordingSink()
+ service = _service(sink)
+ messages = MessageTelemetry(telemetry=service, session_factory=session_factory)
+
+ with caplog.at_level("WARNING"):
+ messages.agent_addressed(
+ tenant_id=TENANT_ZERO,
+ room_id=world.room.id,
+ sender_transport_user_id="@nobody-we-know:test",
+ from_platform=False,
+ agent_metadata=None,
+ agent_live=True,
+ has_attachment=False,
+ )
+ [(_, properties)] = await _reported(messages, service, sink)
+
+ assert properties["sender_kind"] == "unknown"
+ assert "found no client for a message sender" in caplog.text
+
+
+class TestShutdown:
+ async def test_events_after_shutdown_are_counted_not_silently_lost(
+ self, caplog: pytest.LogCaptureFixture
+ ) -> None:
+ sink = _RecordingSink()
+ service = _service(sink)
+ messages = MessageTelemetry(
+ telemetry=service,
+ session_factory=None, # type: ignore[arg-type]
+ )
+ await messages.aclose()
+
+ with caplog.at_level("WARNING"):
+ for _ in range(3):
+ messages.observe(
+ ParticipantMessage(
+ tenant_id=TENANT_ZERO,
+ room_id="room",
+ sender_client_id="someone",
+ sender_role="human",
+ has_attachment=False,
+ in_thread=False,
+ )
+ )
+
+ late = [
+ r
+ for r in caplog.records
+ if "after message telemetry shut down" in r.getMessage()
+ ]
+ assert len(late) == 1
+
+ async def test_events_dropped_during_the_drain_are_all_counted(
+ self, caplog: pytest.LogCaptureFixture
+ ) -> None:
+ """Senders keep running while shutdown drains the queue. The warning is
+ rate-limited, so the drops after its first line are tallied when the
+ drain ends rather than never."""
+ release = asyncio.Event()
+
+ async def _slow_room(tenant_id: str, room_id: str) -> _RoomFacts:
+ await release.wait()
+ return await _a_slack_room(tenant_id, room_id)
+
+ sink = _RecordingSink()
+ service = _service(sink)
+ messages = MessageTelemetry(
+ telemetry=service,
+ session_factory=None, # type: ignore[arg-type]
+ )
+ messages._load_room_facts = _slow_room # type: ignore[method-assign]
+ messages.observe(_said_in("room"))
+ with caplog.at_level("WARNING"):
+ closing = asyncio.create_task(messages.aclose())
+ await asyncio.sleep(0)
+ for _ in range(3):
+ messages.observe(_said_in("room"))
+ release.set()
+ await closing
+
+ late = [
+ r.getMessage()
+ for r in caplog.records
+ if "after message telemetry shut down" in r.getMessage()
+ ]
+ assert len(late) == 2
+ assert "2 more message event(s)" in late[1]
+
+ async def test_closing_stays_inside_the_callers_timeout(
+ self, caplog: pytest.LogCaptureFixture
+ ) -> None:
+ """The worker is cancelled mid-lookup and its cleanup outlasts the
+ shutdown budget. The caller's timeout must still fire: swallowing its
+ cancellation would let shutdown run past the forced-exit grace."""
+ sink = _RecordingSink()
+ service = _service(sink)
+ messages = MessageTelemetry(
+ telemetry=service,
+ session_factory=None, # type: ignore[arg-type]
+ )
+ looking_up = asyncio.Event()
+
+ async def _slow_to_cancel(tenant_id: str, room_id: str) -> None:
+ looking_up.set()
+ try:
+ await asyncio.sleep(10)
+ except asyncio.CancelledError:
+ await asyncio.sleep(1.0)
+ raise
+
+ messages._load_room_facts = _slow_to_cancel # type: ignore[method-assign]
+ messages.observe(
+ ParticipantMessage(
+ tenant_id=TENANT_ZERO,
+ room_id="room",
+ sender_client_id="someone",
+ sender_role="human",
+ has_attachment=False,
+ in_thread=False,
+ )
+ )
+ await looking_up.wait()
+
+ started = time.monotonic()
+ with caplog.at_level("WARNING"), pytest.raises(TimeoutError):
+ async with asyncio.timeout(0.7):
+ await messages.aclose()
+ elapsed = time.monotonic() - started
+
+ assert elapsed < 0.9
+ # The one the worker was cancelled in the middle of is not in the
+ # queue any more, and is still lost.
+ assert "1 message event(s) still queued at shutdown were dropped" in (
+ caplog.text
+ )
+ assert messages._worker is not None
+ await asyncio.wait({messages._worker})
+
+
+class TestOff:
+ def test_enabled_follows_the_service(self) -> None:
+ """What a caller with telemetry-only work checks before doing it."""
+ for enabled in (True, False):
+ messages = MessageTelemetry(
+ telemetry=_service(_RecordingSink(), enabled=enabled),
+ session_factory=None, # type: ignore[arg-type]
+ )
+ assert messages.enabled is enabled
+
+ async def test_nothing_is_queued_or_looked_up_when_telemetry_is_off(
+ self, session_factory: async_sessionmaker[AsyncSession]
+ ) -> None:
+ world = await _world(session_factory)
+ sink = _RecordingSink()
+ service = _service(sink, enabled=False)
+ messages = MessageTelemetry(telemetry=service, session_factory=session_factory)
+
+ messages.observe(_said(world, world.humans[0].id, "human"))
+ messages.agent_addressed(
+ tenant_id=TENANT_ZERO,
+ room_id=world.room.id,
+ sender_transport_user_id=world.humans[0].transport_user_id,
+ from_platform=False,
+ agent_metadata=None,
+ agent_live=False,
+ has_attachment=False,
+ )
+
+ assert messages._worker is None
+ assert await _reported(messages, service, sink) == []
diff --git a/core/tests/switch_core/telemetry/test_relay_attribute_cap.py b/core/tests/switch_core/telemetry/test_relay_attribute_cap.py
index 6d9561f15..34ca456cd 100644
--- a/core/tests/switch_core/telemetry/test_relay_attribute_cap.py
+++ b/core/tests/switch_core/telemetry/test_relay_attribute_cap.py
@@ -37,7 +37,7 @@ class _Capture:
def __init__(self) -> None:
self.sent: list[TelemetryRecord] = []
- async def send(self, record: TelemetryRecord) -> None:
+ def send(self, record: TelemetryRecord) -> None:
self.sent.append(record)
async def aclose(self) -> None:
@@ -89,9 +89,14 @@ async def _attributes_on_the_wire(
http = httpx.AsyncClient(transport=httpx.MockTransport(_handle))
[record] = capture.sent
- await OtlpRelaySink(client=OtlpClient("https://relay.example", 5, {}, http)).send(
- record
+ sink = OtlpRelaySink(
+ client=OtlpClient("https://relay.example", 5, {}, http),
+ flush_interval_seconds=60.0,
+ max_batch=200,
+ max_buffered=10_000,
)
+ sink.send(record)
+ await sink.aclose()
await http.aclose()
return body["resourceLogs"][0]["scopeLogs"][0]["logRecords"][0]["attributes"]
diff --git a/core/tests/switch_core/telemetry/test_service_and_sink.py b/core/tests/switch_core/telemetry/test_service_and_sink.py
index 172845554..e98c91f14 100644
--- a/core/tests/switch_core/telemetry/test_service_and_sink.py
+++ b/core/tests/switch_core/telemetry/test_service_and_sink.py
@@ -9,17 +9,35 @@ import httpx
import pytest
from switch_core.observability.otlp import OtlpClient
+from switch_core.telemetry import sink as sink_module
from switch_core.telemetry.catalogue import TelemetryCatalogueError
from switch_core.telemetry.service import TelemetryService, emit_safely
from switch_core.telemetry.sink import NullSink, OtlpRelaySink, TelemetryRecord
+def _relay_sink(
+ http: httpx.AsyncClient,
+ *,
+ flush_interval_seconds: float = 60.0,
+ max_batch: int = 200,
+ max_buffered: int = 10_000,
+) -> OtlpRelaySink:
+ """A relay sink whose timer will not fire inside a test unless asked to:
+ a test posts by closing it, or by filling a batch."""
+ return OtlpRelaySink(
+ client=OtlpClient("https://relay.example", 5, {}, http),
+ flush_interval_seconds=flush_interval_seconds,
+ max_batch=max_batch,
+ max_buffered=max_buffered,
+ )
+
+
class _RecordingSink:
def __init__(self) -> None:
self.sent: list[TelemetryRecord] = []
self.closed = False
- async def send(self, record: TelemetryRecord) -> None:
+ def send(self, record: TelemetryRecord) -> None:
self.sent.append(record)
async def aclose(self) -> None:
@@ -156,7 +174,7 @@ class TestTagging:
class TestFailuresDoNotReachTheCaller:
async def test_a_sink_that_raises_does_not_surface(self) -> None:
class _Broken:
- async def send(self, record: TelemetryRecord) -> None:
+ def send(self, record: TelemetryRecord) -> None:
raise RuntimeError("relay on fire")
async def aclose(self) -> None:
@@ -197,8 +215,8 @@ class TestTheWireFormat:
return handler(request) # type: ignore[operator]
http = httpx.AsyncClient(transport=httpx.MockTransport(_handle))
- sink = OtlpRelaySink(client=OtlpClient("https://relay.example", 5, {}, http))
- await sink.send(
+ sink = _relay_sink(http)
+ sink.send(
TelemetryRecord(
name="switch_core.usage_snapshot",
properties={"room_count": 7, "from_template": True, "kind": "user"},
@@ -211,6 +229,7 @@ class TestTheWireFormat:
timestamp_ns=1_700_000_000_000_000_000,
)
)
+ await sink.aclose()
await http.aclose()
return captured
@@ -250,8 +269,8 @@ class TestTheWireFormat:
return httpx.Response(200)
http = httpx.AsyncClient(transport=httpx.MockTransport(_handle))
- sink = OtlpRelaySink(client=OtlpClient("https://relay.example", 5, {}, http))
- await sink.send(
+ sink = _relay_sink(http)
+ sink.send(
TelemetryRecord(
name="switch_core.connector_added",
properties={
@@ -267,6 +286,7 @@ class TestTheWireFormat:
timestamp_ns=1_700_000_000_000_000_000,
)
)
+ await sink.aclose()
await http.aclose()
attributes = {a["key"]: a["value"] for a in self._record(body)["attributes"]}
@@ -303,8 +323,9 @@ class TestTheWireFormat:
return httpx.Response(200)
http = httpx.AsyncClient(transport=httpx.MockTransport(_handle))
- sink = OtlpRelaySink(client=OtlpClient("https://relay.example", 5, {}, http))
- await sink.send(_a_record())
+ sink = _relay_sink(http)
+ sink.send(_a_record())
+ await sink.aclose()
await http.aclose()
assert seen == ["https://relay.example/v1/logs"]
@@ -318,8 +339,9 @@ class TestTheWireFormat:
return httpx.Response(200)
http = httpx.AsyncClient(transport=httpx.MockTransport(_handle))
- sink = OtlpRelaySink(client=OtlpClient("https://relay.example", 5, {}, http))
- await sink.send(_a_record())
+ sink = _relay_sink(http)
+ sink.send(_a_record())
+ await sink.aclose()
await http.aclose()
assert "authorization" not in seen
@@ -333,8 +355,9 @@ class TestTheRelayMisbehaving:
http = httpx.AsyncClient(
transport=httpx.MockTransport(handler) # type: ignore[arg-type]
)
- sink = OtlpRelaySink(client=OtlpClient("https://relay.example", 5, {}, http))
- await sink.send(_a_record())
+ sink = _relay_sink(http)
+ sink.send(_a_record())
+ await sink.aclose()
await http.aclose()
async def test_a_refusal_is_swallowed(self) -> None:
@@ -360,11 +383,41 @@ class TestTheRelayMisbehaving:
with caplog.at_level("WARNING"):
await self._send_against(
lambda r: httpx.Response(
- 200, json={"partialSuccess": {"rejectedLogRecords": "3"}}
+ 200, json={"partialSuccess": {"rejectedLogRecords": "1"}}
)
)
- assert "was not sent" in caplog.text
+ assert "rejected 1 of 1" in caplog.text
+
+ async def test_a_partial_success_reports_what_was_lost_not_the_batch(
+ self, caplog: pytest.LogCaptureFixture
+ ) -> None:
+ """One rejected record out of a batch is one lost, not the batch:
+ reporting the whole batch as unsent misstates the loss and sends an
+ operator after the wrong problem."""
+ http = httpx.AsyncClient(
+ transport=httpx.MockTransport(
+ lambda r: httpx.Response(
+ 200,
+ json={
+ "partialSuccess": {
+ "rejectedLogRecords": "1",
+ "errorMessage": "too many attributes",
+ }
+ },
+ )
+ )
+ )
+ sink = _relay_sink(http)
+ for _ in range(3):
+ sink.send(_a_record())
+ with caplog.at_level("WARNING"):
+ await sink.aclose()
+ await http.aclose()
+
+ assert "rejected 1 of 3" in caplog.text
+ assert "too many attributes" in caplog.text
+ assert "was not sent" not in caplog.text
async def test_an_empty_200_is_not_treated_as_a_rejection(
self, caplog: pytest.LogCaptureFixture
@@ -389,35 +442,404 @@ def _a_record() -> TelemetryRecord:
)
-class TestNullSink:
- async def test_it_discards_and_closes(self) -> None:
- sink = NullSink()
- await sink.send(TelemetryRecord("switch_core.x", {}, {}, 1))
+class TestBatching:
+ """The message events fire once per message, so the relay sink posts many
+ events per request rather than one request per event."""
+
+ async def test_events_wait_and_go_together(self) -> None:
+ bodies: list[dict] = []
+
+ def _handle(request: httpx.Request) -> httpx.Response:
+ bodies.append(json.loads(request.content))
+ return httpx.Response(200)
+
+ http = httpx.AsyncClient(transport=httpx.MockTransport(_handle))
+ sink = _relay_sink(http)
+ for name in ("switch_core.room_message_sent", "switch_core.agent_message_sent"):
+ sink.send(TelemetryRecord(name, {}, _a_record().resource, 1))
+
+ assert bodies == []
+ await sink.aclose()
+ await http.aclose()
+
+ assert len(bodies) == 1
+ records = bodies[0]["resourceLogs"][0]["scopeLogs"][0]["logRecords"]
+ # Each record keeps its own name, in both places the relay reads it.
+ assert [r["eventName"] for r in records] == [
+ "switch_core.room_message_sent",
+ "switch_core.agent_message_sent",
+ ]
+ assert [r["body"]["stringValue"] for r in records] == [
+ r["eventName"] for r in records
+ ]
+
+ async def test_a_full_batch_goes_without_waiting_for_the_timer(self) -> None:
+ posted = asyncio.Event()
+ sizes: list[int] = []
+
+ def _handle(request: httpx.Request) -> httpx.Response:
+ body = json.loads(request.content)
+ sizes.append(len(body["resourceLogs"][0]["scopeLogs"][0]["logRecords"]))
+ posted.set()
+ return httpx.Response(200)
+
+ http = httpx.AsyncClient(transport=httpx.MockTransport(_handle))
+ sink = _relay_sink(http, max_batch=3)
+ for _ in range(3):
+ sink.send(_a_record())
+
+ async with asyncio.timeout(2):
+ await posted.wait()
await sink.aclose()
+ await http.aclose()
+
+ assert sizes == [3]
+
+ async def test_events_arriving_mid_post_wait_for_the_next_flush(self) -> None:
+ """A flush posts what it found and no more. Draining until empty would,
+ under steady traffic, post back to back in batches of a few events and
+ never return to waiting — the per-event request load batching is for."""
+ sizes: list[int] = []
+ in_flight = asyncio.Event()
+ release = asyncio.Event()
+
+ async def _handle(request: httpx.Request) -> httpx.Response:
+ body = json.loads(request.content)
+ sizes.append(len(body["resourceLogs"][0]["scopeLogs"][0]["logRecords"]))
+ in_flight.set()
+ await release.wait()
+ return httpx.Response(200)
+
+ http = httpx.AsyncClient(transport=httpx.MockTransport(_handle))
+ sink = _relay_sink(http, max_batch=2)
+ sink.send(_a_record())
+ sink.send(_a_record())
+ async with asyncio.timeout(2):
+ await in_flight.wait()
+ sink.send(_a_record())
+ release.set()
+ await asyncio.sleep(0.05)
+
+ assert sizes == [2]
+ await sink.aclose()
+ await http.aclose()
+ assert sizes == [2, 1]
+
+ async def test_the_timer_posts_a_partial_batch(self) -> None:
+ posted = asyncio.Event()
+
+ def _handle(request: httpx.Request) -> httpx.Response:
+ posted.set()
+ return httpx.Response(200)
+
+ http = httpx.AsyncClient(transport=httpx.MockTransport(_handle))
+ sink = _relay_sink(http, flush_interval_seconds=0.01)
+ sink.send(_a_record())
+
+ async with asyncio.timeout(2):
+ await posted.wait()
+ await sink.aclose()
+ await http.aclose()
+
+ async def test_a_full_buffer_drops_rather_than_grows(
+ self, caplog: pytest.LogCaptureFixture
+ ) -> None:
+ bodies: list[dict] = []
+
+ def _handle(request: httpx.Request) -> httpx.Response:
+ bodies.append(json.loads(request.content))
+ return httpx.Response(200)
+
+ http = httpx.AsyncClient(transport=httpx.MockTransport(_handle))
+ sink = _relay_sink(http, max_batch=100, max_buffered=2)
+ with caplog.at_level("WARNING"):
+ for _ in range(3):
+ sink.send(_a_record())
+ await sink.aclose()
+ await http.aclose()
+
+ assert "buffer is full" in caplog.text
+ sent = [
+ r
+ for body in bodies
+ for r in body["resourceLogs"][0]["scopeLogs"][0]["logRecords"]
+ ]
+ assert len(sent) == 2
+
+ async def test_drops_after_the_last_warning_are_reported_at_shutdown(
+ self, caplog: pytest.LogCaptureFixture
+ ) -> None:
+ """The warning is rate-limited, so drops after it would otherwise never
+ be logged if no later drop came along to carry them."""
+ http = httpx.AsyncClient(
+ transport=httpx.MockTransport(lambda r: httpx.Response(200))
+ )
+ sink = _relay_sink(http, max_batch=100, max_buffered=1)
+ sink.send(_a_record())
+ with caplog.at_level("WARNING"):
+ for _ in range(4):
+ sink.send(_a_record())
+ await sink.aclose()
+ await http.aclose()
+
+ warnings = [
+ r.getMessage() for r in caplog.records if "buffer is full" in r.getMessage()
+ ]
+ assert len(warnings) == 2
+ assert "1 event(s) dropped" in warnings[0]
+ assert "3 event(s) dropped" in warnings[1]
+
+ async def test_a_batch_that_cannot_be_built_costs_only_itself(
+ self, caplog: pytest.LogCaptureFixture
+ ) -> None:
+ """A flush takes the whole buffer at once, so one bad batch must not
+ take the batches after it down with it."""
+ posted: list[int] = []
+
+ def _handle(request: httpx.Request) -> httpx.Response:
+ body = json.loads(request.content)
+ posted.append(len(body["resourceLogs"][0]["scopeLogs"][0]["logRecords"]))
+ return httpx.Response(200)
+
+ http = httpx.AsyncClient(transport=httpx.MockTransport(_handle))
+ sink = _relay_sink(http, max_batch=1)
+ post = sink._post
+ calls = 0
+
+ async def _first_one_breaks(batch: object) -> None:
+ nonlocal calls
+ calls += 1
+ if calls == 1:
+ raise ValueError("bug in building the batch")
+ await post(batch) # type: ignore[arg-type]
+
+ sink._post = _first_one_breaks # type: ignore[method-assign]
+ for _ in range(3):
+ sink.send(_a_record())
+ with caplog.at_level("WARNING"):
+ await sink.aclose()
+ await http.aclose()
+
+ assert posted == [1, 1]
+ assert "batch of 1 event(s) could not be built" in caplog.text
+
+ async def test_events_after_shutdown_are_warned_about_once(
+ self, caplog: pytest.LogCaptureFixture
+ ) -> None:
+ http = httpx.AsyncClient(
+ transport=httpx.MockTransport(lambda r: httpx.Response(200))
+ )
+ sink = _relay_sink(http)
+ await sink.aclose()
+ with caplog.at_level("WARNING"):
+ for _ in range(3):
+ sink.send(_a_record())
+ await http.aclose()
+
+ late = [
+ r
+ for r in caplog.records
+ if "arrived after shutdown began" in r.getMessage()
+ ]
+ assert len(late) == 1
+
+ async def test_events_dropped_during_the_final_flush_are_all_counted(
+ self, caplog: pytest.LogCaptureFixture
+ ) -> None:
+ """Senders keep running while shutdown waits on the last POST. The
+ warning is rate-limited, so the drops after its first line are tallied
+ when the flush ends rather than never."""
+ release = asyncio.Event()
+
+ async def _slow(request: httpx.Request) -> httpx.Response:
+ await release.wait()
+ return httpx.Response(200)
+
+ http = httpx.AsyncClient(transport=httpx.MockTransport(_slow))
+ sink = _relay_sink(http)
+ sink.send(_a_record())
+ with caplog.at_level("WARNING"):
+ closing = asyncio.create_task(sink.aclose())
+ await asyncio.sleep(0)
+ for _ in range(3):
+ sink.send(_a_record())
+ release.set()
+ await closing
+ await http.aclose()
+
+ late = [
+ r.getMessage()
+ for r in caplog.records
+ if "arrived after shutdown began" in r.getMessage()
+ ]
+ assert len(late) == 2
+ assert "2 more telemetry event(s)" in late[1]
+
+ async def test_a_failed_batch_names_what_it_lost(
+ self, caplog: pytest.LogCaptureFixture
+ ) -> None:
+ http = httpx.AsyncClient(
+ transport=httpx.MockTransport(lambda r: httpx.Response(503))
+ )
+ sink = _relay_sink(http)
+ sink.send(_a_record())
+ with caplog.at_level("WARNING"):
+ await sink.aclose()
+ await http.aclose()
+
+ assert "switch_core.deployment_started" in caplog.text
class TestShutdown:
- async def test_in_flight_sends_are_awaited(self) -> None:
- """The events worth losing least are the ones emitted just before the
- process goes away."""
- started = asyncio.Event()
+ """The last flush runs under a fraction of a second. What it cannot post is
+ counted, and a slow request already in flight does not hold the rest."""
+
+ async def test_a_slow_post_in_flight_does_not_hold_back_the_rest(
+ self, caplog: pytest.LogCaptureFixture
+ ) -> None:
+ in_flight = asyncio.Event()
+ posted: list[int] = []
+
+ async def _handle(request: httpx.Request) -> httpx.Response:
+ if not in_flight.is_set():
+ in_flight.set()
+ await asyncio.sleep(10)
+ body = json.loads(request.content)
+ posted.append(len(body["resourceLogs"][0]["scopeLogs"][0]["logRecords"]))
+ return httpx.Response(200)
- class _Slow:
- def __init__(self) -> None:
- self.finished = False
+ http = httpx.AsyncClient(transport=httpx.MockTransport(_handle))
+ sink = _relay_sink(http, max_batch=1)
+ sink.send(_a_record())
+ async with asyncio.timeout(2):
+ await in_flight.wait()
+ sink.send(_a_record())
+ sink.send(_a_record())
+
+ with caplog.at_level("WARNING"), pytest.raises(TimeoutError):
+ async with asyncio.timeout(0.3):
+ await sink.aclose()
+ await http.aclose()
- async def send(self, record: TelemetryRecord) -> None:
- started.set()
- await asyncio.sleep(0.05)
- self.finished = True
+ assert posted == [1, 1]
+ assert (
+ "1 telemetry event(s) were never sent: shutdown ran out of time"
+ in caplog.text
+ )
- async def aclose(self) -> None:
- return None
+ async def test_what_the_timeout_cuts_off_is_counted(
+ self, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
+ ) -> None:
+ """Both the batches still waiting and the ones abandoned mid-request:
+ a batch leaves the buffer before it is posted, so counting only what
+ is left would report nothing lost for a request the timeout cut off."""
+ monkeypatch.setattr(sink_module, "_SHUTDOWN_CONCURRENCY", 2)
+
+ async def _hang(request: httpx.Request) -> httpx.Response:
+ await asyncio.sleep(10)
+ return httpx.Response(200)
+
+ http = httpx.AsyncClient(transport=httpx.MockTransport(_hang))
+ sink = _relay_sink(http, max_batch=2)
+ for _ in range(5):
+ sink.send(_a_record())
+
+ with caplog.at_level("WARNING"), pytest.raises(TimeoutError):
+ async with asyncio.timeout(0.2):
+ await sink.aclose()
+ await http.aclose()
+
+ assert "5 telemetry event(s) were never sent" in caplog.text
+
+ async def test_the_drop_tallies_survive_the_timeout(
+ self, caplog: pytest.LogCaptureFixture
+ ) -> None:
+ async def _hang(request: httpx.Request) -> httpx.Response:
+ await asyncio.sleep(10)
+ return httpx.Response(200)
+
+ http = httpx.AsyncClient(transport=httpx.MockTransport(_hang))
+ sink = _relay_sink(http, max_batch=100, max_buffered=1)
+ sink.send(_a_record())
+ with caplog.at_level("WARNING"):
+ for _ in range(3):
+ sink.send(_a_record())
+ with pytest.raises(TimeoutError):
+ async with asyncio.timeout(0.2):
+ await sink.aclose()
+ await http.aclose()
+
+ full = [
+ r.getMessage() for r in caplog.records if "buffer is full" in r.getMessage()
+ ]
+ assert "2 event(s) dropped" in full[-1]
+
+ async def test_a_relay_that_refuses_at_shutdown_is_not_fed_the_rest(
+ self, caplog: pytest.LogCaptureFixture
+ ) -> None:
+ requests = 0
+
+ def _refuse(request: httpx.Request) -> httpx.Response:
+ nonlocal requests
+ requests += 1
+ return httpx.Response(503)
+
+ http = httpx.AsyncClient(transport=httpx.MockTransport(_refuse))
+ sink = _relay_sink(http, max_batch=1)
+ for _ in range(10):
+ sink.send(_a_record())
+ with caplog.at_level("WARNING"):
+ await sink.aclose()
+ await http.aclose()
+
+ assert requests <= sink_module._SHUTDOWN_CONCURRENCY
+ assert (
+ f"{10 - requests} telemetry event(s) were never sent: the relay "
+ "refused a batch at shutdown"
+ ) in caplog.text
+
+ async def test_a_clean_shutdown_reports_nothing_lost(
+ self, caplog: pytest.LogCaptureFixture
+ ) -> None:
+ http = httpx.AsyncClient(
+ transport=httpx.MockTransport(lambda r: httpx.Response(200))
+ )
+ sink = _relay_sink(http, max_batch=1)
+ for _ in range(10):
+ sink.send(_a_record())
+ with caplog.at_level("WARNING"):
+ await sink.aclose()
+ await http.aclose()
+
+ assert "never sent" not in caplog.text
+
+
+class TestNullSink:
+ async def test_it_discards_and_closes(self) -> None:
+ sink = NullSink()
+ sink.send(TelemetryRecord("switch_core.x", {}, {}, 1))
+ await sink.aclose()
- sink = _Slow()
+
+class TestHandOff:
+ async def test_the_sink_has_the_event_as_soon_as_emit_returns(self) -> None:
+ """No task per event: the message events fire once per message, and the
+ sink only buffers, so a task would be pure overhead on the loop."""
+ sink = _RecordingSink()
service = _service(sink)
+
service.emit("deployment_started", tenant_count=1)
- await started.wait()
+
+ assert [record.name for record in sink.sent] == [
+ "switch_core.deployment_started"
+ ]
+
+ async def test_closing_the_service_closes_the_sink(self) -> None:
+ """The sink posts what it still holds when closed: the events worth
+ losing least are the ones emitted just before the process goes away."""
+ sink = _RecordingSink()
+ service = _service(sink)
+
await service.aclose()
- assert sink.finished
+ assert sink.closed
diff --git a/core/tests/switch_core/telemetry/test_snapshot.py b/core/tests/switch_core/telemetry/test_snapshot.py
index 2391a3d04..79396eee1 100644
--- a/core/tests/switch_core/telemetry/test_snapshot.py
+++ b/core/tests/switch_core/telemetry/test_snapshot.py
@@ -718,6 +718,145 @@ class TestMembership:
assert counts.room_users_max == 3
+class TestActivityTiers:
+ """Three nested populations of chat identities: seen in a room at all,
+ said anything, and said something in a room with an agent."""
+
+ async def test_a_poster_whose_room_was_archived_is_still_seen(
+ self, session_factory: async_sessionmaker[AsyncSession]
+ ) -> None:
+ """The tiers must nest, so nobody counts as having posted this week
+ without also counting as seen in a room."""
+ async with session_factory() as session:
+ room = await _room(session, archived=True)
+ agent = await _client(session, "agent")
+ human = await _client(session, "user")
+ await _join(session, agent, room)
+ await _join(session, human, room)
+ await _say(session, room, human, seq=1)
+
+ counts = await _counts(session)
+
+ assert counts.chat_identity_active_7d == 1
+ assert counts.chat_identity_posted_7d == 1
+ assert counts.chat_identity_in_room_count == 1
+
+ async def test_a_poster_who_left_the_room_is_still_seen(
+ self, session_factory: async_sessionmaker[AsyncSession]
+ ) -> None:
+ async with session_factory() as session:
+ room = await _room(session)
+ await _say(session, room, await _client(session, "user"), seq=1)
+
+ counts = await _counts(session)
+
+ assert counts.chat_identity_posted_7d == 1
+ assert counts.chat_identity_in_room_count == 1
+
+ async def test_a_member_who_posted_is_one_person_not_two(
+ self, session_factory: async_sessionmaker[AsyncSession]
+ ) -> None:
+ async with session_factory() as session:
+ room = await _room(session)
+ human = await _client(session, "user")
+ await _join(session, human, room)
+ await _say(session, room, human, seq=1)
+
+ counts = await _counts(session)
+
+ assert counts.chat_identity_in_room_count == 1
+
+ async def test_someone_who_left_and_went_quiet_is_no_longer_seen(
+ self, session_factory: async_sessionmaker[AsyncSession]
+ ) -> None:
+ async with session_factory() as session:
+ room = await _room(session)
+ await _say(
+ session,
+ room,
+ await _client(session, "user"),
+ seq=1,
+ when=NOW - timedelta(days=8),
+ )
+
+ counts = await _counts(session)
+
+ assert counts.chat_identity_in_room_count == 0
+
+ async def test_talking_only_to_people_is_posting_but_not_active(
+ self, session_factory: async_sessionmaker[AsyncSession]
+ ) -> None:
+ async with session_factory() as session:
+ room = await _room(session)
+ speaker = await _client(session, "user")
+ listener = await _client(session, "user")
+ await _join(session, speaker, room)
+ await _join(session, listener, room)
+ await _say(session, room, speaker, seq=1)
+
+ counts = await _counts(session)
+
+ assert counts.chat_identity_in_room_count == 2
+ assert counts.chat_identity_posted_1d == 1
+ assert counts.chat_identity_active_1d == 0
+
+ async def test_a_person_in_two_rooms_is_one_person(
+ self, session_factory: async_sessionmaker[AsyncSession]
+ ) -> None:
+ async with session_factory() as session:
+ human = await _client(session, "user")
+ for _ in range(2):
+ room = await _room(session)
+ await _join(session, human, room)
+ await _say(session, room, human, seq=1)
+
+ counts = await _counts(session)
+
+ assert counts.chat_identity_in_room_count == 1
+ assert counts.chat_identity_posted_1d == 1
+
+ async def test_only_people_in_live_rooms_count(
+ self, session_factory: async_sessionmaker[AsyncSession]
+ ) -> None:
+ async with session_factory() as session:
+ live = await _room(session)
+ archived = await _room(session, archived=True)
+ await _join(session, await _client(session, "user"), live)
+ await _join(session, await _client(session, "user"), archived)
+ await _join(session, await _client(session, "agent"), live)
+ await _join(session, await _client(session, "bridge"), live)
+
+ counts = await _counts(session)
+
+ assert counts.chat_identity_in_room_count == 1
+
+ async def test_posting_is_windowed(
+ self, session_factory: async_sessionmaker[AsyncSession]
+ ) -> None:
+ async with session_factory() as session:
+ room = await _room(session)
+ human = await _client(session, "user")
+ await _join(session, human, room)
+ await _say(session, room, human, seq=1, when=NOW - timedelta(days=3))
+
+ counts = await _counts(session)
+
+ assert counts.chat_identity_posted_1d == 0
+ assert counts.chat_identity_posted_7d == 1
+
+ async def test_arriving_in_a_room_is_not_posting(
+ self, session_factory: async_sessionmaker[AsyncSession]
+ ) -> None:
+ async with session_factory() as session:
+ room = await _room(session)
+ await _arrive(session, room, await _client(session, "user"), seq=1)
+
+ counts = await _counts(session)
+
+ assert counts.chat_identity_in_room_count == 1
+ assert counts.chat_identity_posted_7d == 0
+
+
class TestNewlyActiveRooms:
async def test_a_room_reports_when_a_person_first_speaks_in_it(
self, session_factory: async_sessionmaker[AsyncSession]
diff --git a/core/tests/switch_core/test_main_telemetry_drain.py b/core/tests/switch_core/test_main_telemetry_drain.py
new file mode 100644
index 000000000..ac27ed44b
--- /dev/null
+++ b/core/tests/switch_core/test_main_telemetry_drain.py
@@ -0,0 +1,91 @@
+"""Telemetry at shutdown: bounded, and the sink always gets its turn."""
+
+from __future__ import annotations
+
+import asyncio
+import time
+from types import SimpleNamespace
+
+import pytest
+
+from switch_core import main
+
+
+class _Step:
+ def __init__(self, *, hangs: bool, takes: float = 0.0) -> None:
+ self.hangs = hangs
+ self.takes = takes
+ self.closed = False
+
+ async def aclose(self) -> None:
+ if self.hangs:
+ await asyncio.sleep(10)
+ await asyncio.sleep(self.takes)
+ self.closed = True
+
+
+async def test_a_message_worker_that_overruns_does_not_cost_the_sink_its_flush() -> (
+ None
+):
+ """The sink's final flush carries every kind of event, so the per-message
+ worker using up a shared budget would lose far more than its own queue."""
+ message_telemetry = _Step(hangs=True)
+ telemetry = _Step(hangs=False)
+ http = SimpleNamespace(closed=False)
+
+ async def _close_http() -> None:
+ http.closed = True
+
+ http.aclose = _close_http
+
+ started = time.monotonic()
+ await main._drain_telemetry(telemetry, message_telemetry, http) # type: ignore[arg-type]
+ elapsed = time.monotonic() - started
+
+ assert telemetry.closed
+ assert http.closed
+ assert elapsed < main._TELEMETRY_DRAIN_SECONDS + 0.2
+
+
+async def test_the_sink_has_the_time_the_message_worker_did_not_use() -> None:
+ """A fixed split would cut the sink off at its share however little of
+ the second the worker took."""
+ sink_takes = (
+ main._TELEMETRY_DRAIN_SECONDS - main._MESSAGE_TELEMETRY_DRAIN_SECONDS + 0.2
+ )
+ telemetry = _Step(hangs=False, takes=sink_takes)
+
+ await main._drain_telemetry(
+ telemetry, # type: ignore[arg-type]
+ _Step(hangs=False), # type: ignore[arg-type]
+ None,
+ )
+
+ assert telemetry.closed
+
+
+async def test_the_whole_drain_stays_inside_its_budget() -> None:
+ started = time.monotonic()
+ await main._drain_telemetry(
+ _Step(hangs=True), # type: ignore[arg-type]
+ _Step(hangs=True), # type: ignore[arg-type]
+ None,
+ )
+
+ assert time.monotonic() - started < main._TELEMETRY_DRAIN_SECONDS + 0.2
+
+
+async def test_a_step_that_fails_is_disclosed_and_the_sink_still_closes(
+ caplog: pytest.LogCaptureFixture,
+) -> None:
+ class _Broken:
+ async def aclose(self) -> None:
+ raise RuntimeError("bug in closing")
+
+ telemetry = _Step(hangs=False)
+
+ with caplog.at_level("WARNING"):
+ await main._drain_telemetry(telemetry, _Broken(), None) # type: ignore[arg-type]
+
+ assert telemetry.closed
+ assert "Reporting queued message events at shutdown failed" in caplog.text
diff --git a/core/tests/switch_core/transport/test_postgres_transport.py b/core/tests/switch_core/transport/test_postgres_transport.py
index 394ee2464..9d0b81b0f 100644
--- a/core/tests/switch_core/transport/test_postgres_transport.py
+++ b/core/tests/switch_core/transport/test_postgres_transport.py
@@ -40,6 +40,7 @@ from switch_core.db.stores.message_store import MessageStore
from switch_core.db.stores.room_store import RoomStore
from switch_core.db.stores.usage_store import UsageStore
from switch_core.observability.metrics import MetricsRegistry, install, uninstall
+from switch_core.observability.throttle import WarningThrottle
from switch_core.provisioning.postgres import PostgresProvisioning
from switch_core.tenant_context import current_tenant_id, tenant_scope
from switch_core.transport import (
@@ -51,8 +52,14 @@ from switch_core.transport import (
TransportError,
TransportHandlers,
)
+from switch_core.transport import postgres as postgres_module
from switch_core.transport.ephemeral import EphemeralBus
from switch_core.transport.invites import InviteBus
+from switch_core.transport.observer import (
+ IgnoreParticipantMessages,
+ ParticipantMessage,
+ ParticipantMessageObserver,
+)
from switch_core.transport.postgres import DELIVERY_PAGE, PostgresTransport
from switch_core.transport.room_cache import RoomCacheLimits, RoomDeliveryCache
from tests.conftest import RLSHarness
@@ -213,6 +220,7 @@ def _transport(
ephemeral: EphemeralBus | None = None,
invites: InviteBus | None = None,
actor_role: str = "agent",
+ message_observer: ParticipantMessageObserver | None = None,
) -> PostgresTransport:
"""A transport for `client_id`, acting in `tenant_id`.
@@ -239,6 +247,7 @@ def _transport(
invites=invites or InviteBus(),
ephemeral=ephemeral or EphemeralBus(),
room_cache=_room_cache(session_factory),
+ message_observer=message_observer or IgnoreParticipantMessages(),
)
@@ -354,6 +363,193 @@ class TestMetering:
assert await self._usage(session_factory) == [("messages", client_id, 1)]
+class _Recorder:
+ def __init__(self) -> None:
+ self.seen: list[ParticipantMessage] = []
+
+ def observe(self, message: ParticipantMessage) -> None:
+ self.seen.append(message)
+
+
+class TestTheParticipantMessageObserver:
+ """The observer hears what the tenant is metered for — what a participant
+ chose to say — once per message, and only after it is committed."""
+
+ async def test_a_metered_message_is_observed_with_what_the_transport_knows(
+ self, session_factory: async_sessionmaker[AsyncSession]
+ ) -> None:
+ async with session_factory() as session:
+ room_id, transport_room_id, client_id, user_id = await _make_room(session)
+ await session.commit()
+ recorder = _Recorder()
+ transport = _transport(
+ session_factory,
+ client_id=client_id,
+ user_id=user_id,
+ actor_role="human",
+ message_observer=recorder,
+ )
+
+ await transport.send_message(
+ transport_room_id, "hello", sender_name="a", metered=True
+ )
+
+ assert recorder.seen == [
+ ParticipantMessage(
+ tenant_id=TENANT_ZERO_ID,
+ room_id=room_id,
+ sender_client_id=client_id,
+ sender_role="human",
+ has_attachment=False,
+ in_thread=False,
+ )
+ ]
+
+ async def test_a_reply_in_a_thread_says_so(
+ self, session_factory: async_sessionmaker[AsyncSession]
+ ) -> None:
+ async with session_factory() as session:
+ _, transport_room_id, client_id, user_id = await _make_room(session)
+ await session.commit()
+ recorder = _Recorder()
+ transport = _transport(
+ session_factory,
+ client_id=client_id,
+ user_id=user_id,
+ message_observer=recorder,
+ )
+
+ root = await transport.send_message(
+ transport_room_id, "root", sender_name="a", metered=True
+ )
+ await transport.send_message(
+ transport_room_id,
+ "reply",
+ sender_name="a",
+ metered=True,
+ thread_root_id=root.event_id,
+ )
+
+ assert [m.in_thread for m in recorder.seen] == [False, True]
+
+ async def test_what_switch_posts_on_someones_behalf_is_not_observed(
+ self, session_factory: async_sessionmaker[AsyncSession]
+ ) -> None:
+ async with session_factory() as session:
+ _, transport_room_id, client_id, user_id = await _make_room(session)
+ await session.commit()
+ recorder = _Recorder()
+ transport = _transport(
+ session_factory,
+ client_id=client_id,
+ user_id=user_id,
+ message_observer=recorder,
+ )
+
+ await transport.send_message(
+ transport_room_id, "unavailable", sender_name="a", metered=False
+ )
+ await transport.send_event(
+ transport_room_id, "com.switch.report.tool_call", {"tool": "Bash"}
+ )
+
+ assert recorder.seen == []
+
+ async def test_a_post_with_several_files_is_one_message(
+ self, session_factory: async_sessionmaker[AsyncSession]
+ ) -> None:
+ async with session_factory() as session:
+ _, transport_room_id, client_id, user_id = await _make_room(session)
+ await session.commit()
+ recorder = _Recorder()
+ transport = _transport(
+ session_factory,
+ client_id=client_id,
+ user_id=user_id,
+ message_observer=recorder,
+ )
+
+ for index in range(3):
+ await transport.send_media(
+ transport_room_id,
+ f"blob://{index}",
+ f"file-{index}.txt",
+ "text/plain",
+ 10,
+ sender_name="a",
+ msgtype="m.file",
+ metered=True,
+ group={"id": "g1", "index": index, "total": 3},
+ )
+
+ assert len(recorder.seen) == 1
+ assert recorder.seen[0].has_attachment is True
+
+ async def test_an_observer_that_raises_does_not_unsend_the_message(
+ self,
+ session_factory: async_sessionmaker[AsyncSession],
+ monkeypatch: pytest.MonkeyPatch,
+ caplog: pytest.LogCaptureFixture,
+ ) -> None:
+ """The row is committed before the observer hears of it, so a failure
+ there must not tell the sender its message was lost: the retry would
+ post it twice. Caught, so it is counted, and logged once rather than
+ once per message."""
+
+ class _Broken:
+ def observe(self, message: ParticipantMessage) -> None:
+ raise RuntimeError("observer bug")
+
+ registry = MetricsRegistry()
+ install(registry)
+ monkeypatch.setattr(
+ postgres_module, "_observer_failures", WarningThrottle(60.0)
+ )
+ async with session_factory() as session:
+ room_id, transport_room_id, client_id, user_id = await _make_room(session)
+ await session.commit()
+ transport = _transport(
+ session_factory,
+ client_id=client_id,
+ user_id=user_id,
+ message_observer=_Broken(),
+ )
+
+ try:
+ with caplog.at_level("ERROR"):
+ results = [
+ await transport.send_message(
+ transport_room_id, body, sender_name="a", metered=True
+ )
+ for body in ("still sent", "and this")
+ ]
+ failures = next(
+ p
+ for p in registry.collect()
+ if p.name == "switch.messages.observer_failures"
+ )
+ finally:
+ uninstall()
+
+ assert all(result.event_id for result in results)
+ async with session_factory() as session:
+ stored = await session.scalars(
+ select(Message).where(Message.room_id == room_id).order_by(Message.seq)
+ )
+ assert [m.body for m in stored] == ["still sent", "and this"]
+ assert failures.numbers[0].value == 2.0
+ assert (
+ len(
+ [
+ r
+ for r in caplog.records
+ if "Message observer failed" in r.getMessage()
+ ]
+ )
+ == 1
+ )
+
+
class TestSending:
async def test_a_message_becomes_a_row(
self, session_factory: async_sessionmaker[AsyncSession]
@@ -630,6 +826,7 @@ class TestMedia:
invites=InviteBus(),
ephemeral=EphemeralBus(),
room_cache=_room_cache(session_factory),
+ message_observer=IgnoreParticipantMessages(),
)
diff --git a/deploy/observability/dashboard.json b/deploy/observability/dashboard.json
index 6793ba358..a6e8245dd 100644
--- a/deploy/observability/dashboard.json
+++ b/deploy/observability/dashboard.json
@@ -288,8 +288,8 @@
{
"definition": {
"type": "timeseries",
- "title": "Delivery failures per second",
- "show_legend": false,
+ "title": "Delivery and message-observer failures per second",
+ "show_legend": true,
"requests": [
{
"q": "sum:switch.messages.delivery_failures{$env,$service}.as_rate()",
@@ -297,6 +297,13 @@
"style": {
"palette": "warm"
}
+ },
+ {
+ "q": "sum:switch.messages.observer_failures{$env,$service} by {actor}.as_rate()",
+ "display_type": "bars",
+ "style": {
+ "palette": "orange"
+ }
}
]
},
diff --git a/docs/old/observability.md b/docs/old/observability.md
index 019328adc..4c019ad32 100644
--- a/docs/old/observability.md
+++ b/docs/old/observability.md
@@ -236,6 +236,11 @@ handled in a way that leaves no trace.
- **`switch.messages.send_failures`** — a send that raises before the commit
writes nothing, so the symptom is an absence, and an absence is what a quiet
room looks like too.
+- **`switch.messages.observer_failures`** — reporting a message for product
+ telemetry raised, after the transport committed it or while an agent's
+ consumer was delivering it. Caught, because the message itself is fine: a
+ send that raised after its commit would be retried and posted twice. So the
+ only trace is a rate-limited log line.
- **`switch.agent.events_dropped`** — an agent's buffer discards events when it
overflows or when they age out. The agent is *told* it missed them, so
nothing is hidden from the agent; nothing told anybody else.
diff --git a/docs/old/telemetry-events.md b/docs/old/telemetry-events.md
index cb83ccc5f..c360674b8 100644
--- a/docs/old/telemetry-events.md
+++ b/docs/old/telemetry-events.md
@@ -225,7 +225,7 @@ server setting. The Console instead sends a bespoke `build` attribute
server equivalent. The two are deliberately different fields rather than one
field meaning different things on each side.
-## Three kinds of event
+## Four kinds of event
**A daily snapshot**, one per deployment, carrying the counts that describe the
installation. This answers "how much".
@@ -237,28 +237,42 @@ seconds elapsed since install. This answers "how fast to value".
failure: which platforms are in use, how often a bridge drops, whether one
connector fails repeatedly before it works.
+**Message events**, one per message, the only high-volume kind. These carry how
+much is said and between whom — people to agents, agents back, people among
+themselves — and the size of the room it was said in.
+
The snapshot exists because of the identifier rule. Without room ids,
per-occurrence events can tell you *how much* happened but never *across how
many rooms* — counting distinct anything in Amplitude requires an identifier for
the thing being counted. Counting locally and reporting the total sidesteps it.
-### Why there is no `message_sent` event
-
-Message volume is reported as a count in the daily snapshot, not as an event per
-message. Three reasons, in order of weight:
-
-1. **Volume.** The relay path has no batching, no retry and no queue — one HTTP
- request per event, fire-and-forget, because a desktop app emits a handful an
- hour. A busy server emits thousands of messages an hour. Per-message events
- would be a different class of load on a component not built for it.
-2. **It answers nothing extra.** Without a room id, a per-message event supports
- the same charts the snapshot count does.
-3. **Content risk.** An event shaped around a message is the one most likely to
- grow a property that reveals something about the message. Not having it
- removes the temptation.
-
-If per-message granularity is ever genuinely needed, it should arrive with
-batching on the export path first.
+### Message events, and what they cannot tell you
+
+An earlier version of this note argued against an event per message, for three
+reasons. Each is now answered rather than ignored:
+
+1. **Volume.** The relay path used to post one request per event. The relay
+ sink now batches: events wait up to five seconds, or until two hundred are
+ queued, and go as one request. Its buffer is bounded, so a relay that is down
+ costs events, never memory.
+2. **The sender must not pay for it.** The transport and the agent consumer
+ hand a small record to a queue and return. One worker looks up the room and
+ the sender — cached, so a busy room is read once every five minutes rather
+ than once per message — and emits. That queue is bounded too, and drops with
+ a warning when it falls behind.
+3. **Content risk.** The events carry the same kind of property as every
+ other: closed sets, booleans and counts. There is no property a message body,
+ a name or an id could reach, and the catalogue test enforces it.
+
+**They count messages, not people.** No user id is ever sent, so in Amplitude
+every event from a deployment belongs to one subject — the deployment — and
+Amplitude's own "active users" counts deployments. How many *people* are active
+comes only from the snapshot, which counts distinct people locally and reports
+the total: `chat_identity_in_room_count` (seen in a room at all),
+`chat_identity_posted_*` (said anything) and `chat_identity_active_*` (said
+something in a room with an agent), with `user_active_*` counting the same
+last tier as Switch accounts. The message events say how much those people
+said, and to whom.
## Definitions
@@ -332,6 +346,9 @@ never invisible.
| `chat_identity_count` | number | chat identities — Slack, Mattermost and other platform accounts — that exist, claimed by an account or not |
| `chat_identity_active_1d` | number | distinct chat identities that used a room with an agent in 24h. Most belong to no Switch account and one person may have several, which is why this runs above `user_active_1d`. Servers released before this change reported this figure as `user_active_1d` |
| `chat_identity_active_7d` | number | same over 7 days |
+| `chat_identity_posted_1d` | number | distinct chat identities that said anything in any room in 24h, agent or no agent |
+| `chat_identity_posted_7d` | number | same over 7 days |
+| `chat_identity_in_room_count` | number | distinct chat identities Switch has seen in a room: a member of at least one unarchived room of any origin, or anyone who said anything in the last 7 days. Not a channel's whole audience — see "Tiers of activity" |
| `room_count` | number | **the headline figure** — unarchived rooms a *person* created |
| `room_agent_created_count` | number | unarchived rooms an agent created for itself |
| `room_system_created_count` | number | unarchived channels Switch adopted after being invited to them on a platform |
@@ -384,6 +401,28 @@ participant *said*: chat messages, and commands a person typed on a platform.
The automatic notice Switch posts under an agent's name when it cannot take a
request is excluded as well — the agent did not say it.
+**Tiers of activity.** `chat_identity_in_room_count`,
+`chat_identity_posted_7d` and `chat_identity_active_7d` nest: everyone Switch
+has seen in a room, those who said anything, and those who said something in a
+room with an agent. The nesting holds by construction — anyone who posted in
+the last 7 days counts as seen, even if they have since left the room or it
+was archived. `chat_identity_posted_7d - chat_identity_active_7d` is people
+talking only to each other. All three count chat identities, so someone on both
+Slack and Teams counts twice; only the top tier is also counted as accounts
+(`user_active_*`). "In a room with an agent" stands in for "talked to an
+agent": whether a message addressed one is decided per agent as it is
+delivered, and is not stored where a count can read it.
+`agent_message_received` is the per-message measure of that.
+
+**What "seen in a room" cannot see.** Switch records a person in a room only
+when they post, are added by name, or join the channel after Switch adopted
+it. Nothing lists the members a channel already had, so someone who has only
+ever read an adopted channel is in no count here, and
+`chat_identity_in_room_count - chat_identity_posted_7d` is **not** the passive
+audience: it is the people Switch happens to know about who stayed quiet,
+usually a small fraction of the real one. Counting readers would take a
+member sync on each platform, which no bridge does today.
+
**Turns rather than senders.** A turn is one message classified by who sent the
message *before* it in the same room. That is the only way to tell an agent
answering a person from two agents talking among themselves: a sender-only count
@@ -622,6 +661,79 @@ migrations run in a different event loop from the server, so the answer would
have to be carried across on a module global, and it is operational trivia
rather than something the product wants to know.
+### Message events — one per message
+
+All three come from `telemetry/messages.py`, reported off the sender's path as
+described [above](#message-events-and-what-they-cannot-tell-you). A post with
+several files is one message. Room facts are cached for five minutes, so a
+member count can be that far behind. A room, sender or agent whose lookup
+fails is not looked up again for five seconds, and its events report it as
+`unknown` meanwhile; every other room carries on. Three failures in a row,
+with no lookup succeeding between them, pause every lookup for five seconds.
+Either way a database incident shows in the charts as a burst of `unknown`
+rather than as missing messages.
+
+**`room_message_sent`** — everything a participant chose to say in a room: the
+population the tenant is metered for, reported by the transport after the
+write commits. What Switch posts on a participant's behalf — greetings, command
+replies, offline and refusal notices, a template's kickoff — is not in it.
+Neither is a command a person types (`!reset`, a slash command): commands are
+not metered, although the snapshot's activity counts treat one as something
+the person said.
+
+| Property | Type |
+|---|---|
+| `sender_kind` | `user` \| `agent` — only people and agents are metered, so `platform` does not occur here |
+| `bridge_platform` | platform, `none` for an internal room, `unknown` if the room could not be read |
+| `channel_type` | channel type, or `unknown` if the room could not be read |
+| `room_user_count` | number — human members of the room, `-1` if it could not be read |
+| `room_agent_count` | number — agent members, `-1` if it could not be read |
+| `has_attachment` | boolean |
+| `in_thread` | boolean |
+
+The average of `room_user_count` over these events is the room size a typical
+message is said to — weighted by messages, not by rooms. **Filter out
+`bridge_platform = unknown` before averaging**: those are rooms that could not
+be read, and their `-1` would pull the mean down. Every property is required on
+every event and the catalogue has no empty value, so `-1` is the marker. The
+snapshot's `room_users_mean` is the per-room figure.
+
+**`agent_message_received`** — a message an agent was asked to act on:
+addressed to it (by name, alias, role, or a direct room) *and* let through its
+addressing policy and budget, whether or not the agent was there to take it.
+Reported by the agent's consumer, where that is decided, so a message naming
+two agents is two of these and one `room_message_sent`. Switch's own
+auto-replies are excluded; a request Switch carries on a person's behalf is
+included, as `platform`. A message a hosted agent's mailbox already holds — the
+same message seen again — is not counted twice, and nor is a multi-file post
+whose files arrive too far apart to be delivered together: it is counted with
+its first file. A typed command addressed to an agent is not counted.
+`agent_live` splits being asked from being there: filter on it to count only
+the requests an agent could act on as they arrived.
+
+| Property | Type |
+|---|---|
+| `sender_kind` | `user` \| `agent` \| `platform` \| `unknown` |
+| `known_agent_type` | the receiving agent's runtime |
+| `bridge_platform` | platform |
+| `channel_type` | channel type |
+| `has_attachment` | boolean |
+| `agent_live` | boolean — the agent had a live session for the room when the message arrived. False when it was offline, still starting, or its cloud worker refused the message (stopped, broken, or a full mailbox); always false for a `session_passive` agent, which reads its messages later rather than live |
+
+**`agent_message_sent`** — a message an agent posted: its replies, status
+updates, anything it chose to say. Every one is also a `room_message_sent` with
+`sender_kind = agent`; this one carries the runtime as well. `room_user_count`
+separates an agent answering people from agents talking among themselves.
+
+| Property | Type |
+|---|---|
+| `known_agent_type` | the sending agent's runtime, or `unknown` if the agent could not be looked up |
+| `bridge_platform` | platform |
+| `channel_type` | channel type |
+| `room_user_count` | number, `-1` if the room could not be read |
+| `has_attachment` | boolean |
+| `in_thread` | boolean |
+
### Closed value sets
`bridge_platform`: `slack` | `mattermost` | `discord` | `teams` | `telegram` |
@@ -630,6 +742,13 @@ rather than something the product wants to know.
kept apart because collapsing them would misreport a Slack-bridged room as
internal-only whenever that lookup has a transient error.
+`channel_type`: `channel_public` | `channel_private` | `direct` | `none` |
+`unknown`, with `unknown` meaning what it does for `bridge_platform`. Only the
+message events can carry it.
+
+`sender_kind`: `user` | `agent` | `platform` | `unknown`. `platform` is Switch
+itself speaking; `unknown` a sender whose client could not be looked up.
+
`outcome`: `success` | `failure`. `failure_reason` is an enumerated code per
event, `none` on success — never an exception message.
@@ -729,6 +848,13 @@ the operational export follows. A value carrying the signal path is refused at
startup, because posting to `/v1/logs/v1/logs` would be a 404 the relay reports
once per event into a log nobody is reading.
+Product events are batched: they arrive at the sink up to five seconds after
+they happen, several to a request. Whatever is buffered at shutdown is posted
+on the way out, several batches at once, within a one-second shutdown budget:
+the message worker's drain takes at most 0.4 seconds of it and the sink has
+the rest. What cannot be posted in that time is lost, and the server log says
+how many events that was.
+
The same sink serves the operational export, so one process shows both streams. Every event is printed as it arrives, decoded, with all of its
properties. Create a room, register an
agent, connect a bridge, send a message, and watch. The first snapshot lands
@@ -756,7 +882,7 @@ that tests the vendor's own filtering.
### Every event, and how to make it fire
-Twenty-two events. Work down the list against a local sink
+Work down the list against a local sink
(`python scripts/otlp_sink.py`) and each one prints as it arrives with all of
its properties.
@@ -823,6 +949,17 @@ Two things to set up first, or a third of the list cannot fire at all:
(`bridges/agent/api/activity_routes.py`), with a UUID id. A repeat for the
same agent and session emits nothing, across restarts, for 7 days.
+**Messages** — `telemetry/messages.py`, fed by `transport/postgres.py` and
+`clients/agent_consumer.py`
+
+- `room_message_sent` — any person or agent posting in any room. A post with
+ three files is one event. A command reply or an offline notice is none.
+- `agent_message_sent` — an agent posting, through its MCP tool or the HTTP
+ API. Arrives beside a `room_message_sent` for the same message.
+- `agent_message_received` — tag an agent, or write in a direct room with one.
+ Tag two agents and there are two. Tag an agent whose policy refuses you and
+ there are none.
+
**Connectors** — `bridges/collaboration/lifecycle_service.py`
- `bridge_connected` — a bridge reaching its platform. Also fires with
diff --git a/scripts/otlp_sink.py b/scripts/otlp_sink.py
index 1cf08a56d..f27e36fef 100644
--- a/scripts/otlp_sink.py
+++ b/scripts/otlp_sink.py
@@ -106,9 +106,10 @@ def _summarise_logs(payload: dict[str, Any]) -> list[str]:
for scope in resource.get("scopeLogs", []):
records = scope.get("logRecords", [])
# A product event is a log record carrying an event name, and it
- # wants the opposite treatment from a log line: every property
- # matters and there is one record per request, where a log batch
- # is many records of which the first few are representative.
+ # wants the opposite treatment from a log line: every record and
+ # every property matters, even with many events batched into one
+ # request, where a log batch is many records of which the first
+ # few are representative.
if records and "eventName" in records[0]:
lines.extend(_summarise_events(records))
continue