Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
feat(messaging): handle an inbound platform event once (CHOO-2626)
Petr Bauch9fcffc32fb0d
diff --git a/core/switch_core/bridges/collaboration/install.py b/core/switch_core/bridges/collaboration/install.pyindex aa1b2d6fa..ceb5c16a8 100644--- a/core/switch_core/bridges/collaboration/install.py+++ b/core/switch_core/bridges/collaboration/install.py@@ -139,11 +139,26 @@ class InboundWebhook:is `None` for every real event, and it arrives before any workspace hasinstalled anything — so it must be answerable with no tenant, no installrow, and nothing running.++ `external_event_id` is the platform's own id for this delivery, and the+ only thing two copies of one event have in common — the payload is+ identical, so nothing else could tell a retry from a second message saying+ the same words. `None` means the platform does not number this kind of+ envelope, which on Slack means the kind it also does not retry; it never+ means "this one was not checked".++ `delivery_attempt` is how many times the platform has given up on us and+ sent this again, zero on a first delivery. It changes nothing about how the+ event is handled — the receipt decides that — and is carried because it is+ the only place the deployment is told its own acknowledgements are arriving+ too late."""envelope_type: strpayload: dict[str, Any]handshake: str | None+ external_event_id: str | None+ delivery_attempt: int@dataclass(frozen=True)@@ -245,9 +260,9 @@ class MessagingAppInstaller(ABC):@abstractmethoddef parse_webhook(- self, *, endpoint: WebhookEndpoint, body: bytes+ self, *, endpoint: WebhookEndpoint, headers: Mapping[str, str], body: bytes) -> InboundWebhook:- """Read a verified request body into an event a running adapter takes.+ """Read a verified request into an event a running adapter takes.Called only after :meth:`verify_webhook` has passed, and separate fromit for exactly that reason: parsing before verifying is how an@@ -257,6 +272,11 @@ class MessagingAppInstaller(ABC):method knows the encoding, which is per platform and per endpoint —Slack posts JSON to one of its three and form data to the other two.+ Takes the headers as well as the body because a delivery is described+ in both: the event is in the body, and how many times it has been sent+ is in a header. Which header, and whether there is one, is the+ platform's business rather than the route's.+Raise :class:`WebhookPayloadError` for a body that cannot be read."""diff --git a/core/switch_core/bridges/collaboration/install_routes.py b/core/switch_core/bridges/collaboration/install_routes.pyindex 899ec55fe..fb8818228 100644--- a/core/switch_core/bridges/collaboration/install_routes.py+++ b/core/switch_core/bridges/collaboration/install_routes.py@@ -240,6 +240,20 @@ def create_messaging_install_router(logger.info("Answered a %s URL verification", platform)return PlainTextResponse(event.handshake)+ if event.delivery_attempt > 0:+ # The only signal this deployment gets that its own acknowledgements+ # are arriving too late. The event itself is handled normally — the+ # receipt decides whether it is a duplicate — but a run of these is+ # the platform saying the three-second answer is being missed, and+ # nothing else in the system would say so.+ logger.warning(+ "%s is re-sending a %s event (attempt %s), which means an earlier "+ "delivery was not acknowledged in time",+ platform,+ event.envelope_type,+ event.delivery_attempt,+ )+try:# Before resolving, because this is the one event that arrives as# the bridge it would be resolved to is going away.diff --git a/core/switch_core/bridges/collaboration/install_service.py b/core/switch_core/bridges/collaboration/install_service.pyindex bacbb1fbe..062d3f678 100644--- a/core/switch_core/bridges/collaboration/install_service.py+++ b/core/switch_core/bridges/collaboration/install_service.py@@ -77,6 +77,7 @@ from switch_core.bridges.collaboration.lifecycle_service import (from switch_core.crypto import decrypt_token, encrypt_tokenfrom switch_core.db.models import MessagingInstallfrom switch_core.db.session_scope import tenant_session+from switch_core.db.stores.messaging_event_store import MessagingEventReceiptStorefrom switch_core.db.stores.messaging_install_store import (INSTALL_ACTIVE,INSTALL_DISCONNECTED,@@ -120,9 +121,15 @@ class Revocation:@dataclass(frozen=True)class WebhookTarget:- """Where one verified event goes: a tenant, a bridge, and its live adapter."""+ """Where one verified event goes: a tenant, a bridge, and its live adapter.++ `platform` is carried rather than passed alongside because it is part of+ the answer: the same workspace id could in principle be issued by two+ platforms, and every row written about this delivery is keyed by the pair.+ """tenant_id: str+ platform: strbridge_id: stradapter: CollaborationAdapter@@ -144,6 +151,7 @@ class MessagingInstallService:*,session_factory: async_sessionmaker[AsyncSession],store: MessagingInstallStore,+ receipts: MessagingEventReceiptStore,installers: MessagingInstallerRegistry,lifecycle: CollaborationBridgeLifecycleService,public_origin: str,@@ -151,6 +159,7 @@ class MessagingInstallService:) -> None:self._session_factory = session_factoryself._store = store+ self._receipts = receiptsself._installers = installersself._lifecycle = lifecycleself._public_origin = public_origin@@ -426,7 +435,7 @@ class MessagingInstallService:"""installer = self._installers.get(platform)installer.verify_webhook(headers=headers, body=body)- return installer.parse_webhook(endpoint=endpoint, body=body)+ return installer.parse_webhook(endpoint=endpoint, headers=headers, body=body)def revocation(self, *, platform: str, event: InboundWebhook) -> Revocation | None:"""Whether this event is the platform ending the install, and for whom.@@ -492,20 +501,75 @@ class MessagingInstallService:f"{workspace_id}, is not running")return WebhookTarget(- tenant_id=tenant_id, bridge_id=install.bridge_id, adapter=adapter+ tenant_id=tenant_id,+ platform=platform,+ bridge_id=install.bridge_id,+ adapter=adapter,)async def deliver(self, target: WebhookTarget, event: InboundWebhook) -> None:- """Hand a resolved event to the bridge, as its own transport would.-- With **nothing bound**, which is not an oversight. A bridge that- receives over a socket dispatches from a task that binds no tenant, and- every handler below it binds the tenant of the room it is acting on.- Binding here would make the two delivery paths differ in the one- respect that decides who a message reaches, and would hide a handler- that had forgotten to bind for itself — for exactly as long as it took- someone to receive the same event over a socket instead.+ """Hand a resolved event to the bridge, at most once.++ Claiming comes first and the dispatch second, so a retry that arrives+ while the first delivery is still working finds the event taken and+ stops. That ordering is the whole of the deduplication: the platform+ sends the same event again whenever it is not answered in time, and+ the payload of a retry is identical to the original, so nothing later+ in the stack could tell it from someone saying the same thing twice.++ An event the platform does not number is dispatched without a claim.+ That is not a weaker guarantee quietly accepted — Slack numbers what it+ retries, so an envelope with no id is one that arrives exactly once.++ The dispatch itself runs with **nothing bound**, which is not an+ oversight. A bridge that receives over a socket dispatches from a task+ that binds no tenant, and every handler below it binds the tenant of+ the room it is acting on. Binding here would make the two delivery+ paths differ in the one respect that decides who a message reaches, and+ would hide a handler that had forgotten to bind for itself — for+ exactly as long as it took someone to receive the same event over a+ socket instead."""+ if event.external_event_id is None:+ await self._dispatch(target, event)+ return++ async with tenant_session(self._session_factory, target.tenant_id) as session:+ receipt = await self._receipts.claim(+ session,+ platform=target.platform,+ external_event_id=event.external_event_id,+ )+ if receipt is None:+ logger.info(+ "Dropped a repeat delivery of %s event %s to bridge %s "+ "(attempt %s); it has already been taken",+ target.platform,+ event.external_event_id,+ target.bridge_id,+ event.delivery_attempt,+ )+ return+ receipt_id = receipt.id+ # Before the dispatch, not with it. The index entry this writes is+ # what a concurrent retry collides with, and an uncommitted one+ # makes that retry wait for the turn instead of losing to it.+ await session.commit()++ await self._dispatch(target, event)++ async with tenant_session(self._session_factory, target.tenant_id) as session:+ await self._receipts.mark_handled(session, receipt_id=receipt_id)+ pruned = await self._receipts.prune(session)+ await session.commit()+ if pruned:+ logger.info(+ "Pruned %s expired messaging event receipts for tenant %s",+ pruned,+ target.tenant_id,+ )++ async def _dispatch(self, target: WebhookTarget, event: InboundWebhook) -> None:with no_tenant():await target.adapter.dispatch_event(envelope_type=event.envelope_type, payload=event.payloaddiff --git a/core/switch_core/bridges/collaboration/slack/install.py b/core/switch_core/bridges/collaboration/slack/install.pyindex 848aada72..09beb5119 100644--- a/core/switch_core/bridges/collaboration/slack/install.py+++ b/core/switch_core/bridges/collaboration/slack/install.py@@ -65,6 +65,30 @@ def _json_object(raw: bytes) -> dict[str, Any]:return parsed+#: Slack's count of how many times it has re-sent a delivery, absent on the+#: first. Header names arrive from Starlette lower-cased and are compared that+#: way; Slack's own spelling is `X-Slack-Retry-Num`.+_RETRY_NUM_HEADER = "x-slack-retry-num"+++def _retry_number(headers: Mapping[str, str]) -> int:+ """How many times Slack has sent this already, defaulting to none.++ Unauthenticated in the sense that it is not covered by the signature — the+ signature is over the body and the timestamp — so it is read as a hint and+ never as a decision. A forged value cannot make an event be handled twice+ or dropped, because the receipt decides that; the worst it can do is put a+ wrong number in a log line.+ """+ raw = headers.get(_RETRY_NUM_HEADER)+ if raw is None:+ return 0+ try:+ return int(raw)+ except ValueError:+ return 0++def _form_fields(raw: bytes) -> dict[str, Any]:"""Slack's form encoding as a flat dict, matching Socket Mode's payload.@@ -215,8 +239,10 @@ class SlackAppInstaller(MessagingAppInstaller):raise WebhookAuthenticityError("bad Slack signature")def parse_webhook(- self, *, endpoint: WebhookEndpoint, body: bytes+ self, *, endpoint: WebhookEndpoint, headers: Mapping[str, str], body: bytes) -> InboundWebhook:+ attempt = _retry_number(headers)+if endpoint == "commands":# A slash command posts its fields as a form, and Socket Mode# delivers that same flat dict — so the parsed form *is* the@@ -225,6 +251,13 @@ class SlackAppInstaller(MessagingAppInstaller):envelope_type="slash_commands",payload=_form_fields(body),handshake=None,+ # Slack retries neither of the two form endpoints. A command+ # and an interaction are a person waiting on a dialog, and a+ # reply that arrives a minute late is worse than none — so+ # Slack sends each exactly once and there is no id on it to+ # deduplicate by.+ external_event_id=None,+ delivery_attempt=attempt,)if endpoint == "interactive":@@ -239,6 +272,8 @@ class SlackAppInstaller(MessagingAppInstaller):envelope_type="interactive",payload=_json_object(raw.encode()),handshake=None,+ external_event_id=None,+ delivery_attempt=attempt,)envelope = _json_object(body)@@ -253,11 +288,25 @@ class SlackAppInstaller(MessagingAppInstaller):"Slack sent a URL verification with no challenge to echo")return InboundWebhook(- envelope_type="url_verification", payload=envelope, handshake=challenge+ envelope_type="url_verification",+ payload=envelope,+ handshake=challenge,+ external_event_id=None,+ delivery_attempt=attempt,)+ event_id = envelope.get("event_id")return InboundWebhook(- envelope_type="events_api", payload=envelope, handshake=None+ envelope_type="events_api",+ payload=envelope,+ handshake=None,+ # The one envelope Slack numbers, and the one it retries. A+ # retried delivery carries the same `event_id` as the original,+ # which is the whole basis of handling it once.+ external_event_id=event_id+ if isinstance(event_id, str) and event_id+ else None,+ delivery_attempt=attempt,)def workspace_of_event(self, payload: Mapping[str, object]) -> str:diff --git a/core/switch_core/db/models.py b/core/switch_core/db/models.pyindex 2099d6ed6..caa5dc649 100644--- a/core/switch_core/db/models.py+++ b/core/switch_core/db/models.py@@ -1280,6 +1280,72 @@ class MessagingInstallState(TenantScoped, Base):)+class MessagingEventReceipt(TenantScoped, Base):+ """One inbound platform event, claimed once so it is handled once.++ Platforms deliver at least once. Slack gives three seconds to acknowledge+ an event and retries what it does not get an answer to — so a deployment+ under load, or one restarted mid-request, is told the same thing again.+ Without a record of what has been taken, the second telling produces a+ second answer in the customer's channel, which is the visible failure: an+ agent replying twice to one question.++ **The row is written before the work, not after**, and the unique index is+ what arbitrates. Two retries in flight at once both reach the insert and+ exactly one survives it; the loser stops there. Recording afterwards would+ order the two the wrong way round — both would dispatch, and the duplicate+ would be detected once it no longer mattered.++ That ordering chooses at-most-once over at-least-once, which is worth+ stating plainly: an event claimed by a process that then dies is not+ retried, because the platform has already been told 200 and this table says+ the event is taken. It is not a new loss. The route has acknowledged before+ handling since it was written — it has to, the deadline is shorter than a+ turn — so the event was already unrecoverable at that point. What this adds+ is `handled_at`, which makes the loss visible: a claimed row that never+ completed is a real event that reached nobody, and it can be found.++ `external_event_id` is the platform's own id for the delivery, and only+ some envelopes have one. Slack numbers Events API envelopes and retries+ only those; a slash command and an interaction get one shot and no id, so+ there is nothing to deduplicate and no row here. A missing id means "the+ platform does not retry this", not "this was not checked".++ Uniqueness is `(tenant_id, platform, external_event_id)` and not the+ deployment-wide pair, unlike the workspace claim on `messaging_installs`.+ The two would be equivalent — an event id is unique in the platform's own+ namespace and a workspace belongs to one tenant — so the tenant-local index+ is the one to prefer: it keeps one customer's event ids out of another's+ namespace entirely, and it means a conflict is always with a row the+ inserting tenant can actually see rather than an opaque refusal naming+ somebody else's.+ """++ __tablename__ = "messaging_event_receipts"+ __table_args__ = (+ UniqueConstraint(+ "tenant_id",+ "platform",+ "external_event_id",+ name="uq_messaging_event_receipts_event",+ ),+ # Pruning reads this and nothing else. Receipts are only useful for as+ # long as the platform might still retry, and the table would otherwise+ # grow with every message the busiest workspace ever sends.+ Index("ix_messaging_event_receipts_received_at", "received_at"),+ )++ id: Mapped[str] = mapped_column(Text, primary_key=True, default=_uuid)+ platform: Mapped[str] = mapped_column(Text, nullable=False)+ external_event_id: Mapped[str] = mapped_column(Text, nullable=False)+ received_at: Mapped[datetime] = mapped_column(+ DateTime(timezone=True), server_default=func.now(), nullable=False+ )+ handled_at: Mapped[datetime | None] = mapped_column(+ DateTime(timezone=True), nullable=True+ )++# ── Server-Side Connectors ────────────────────────────────────────────────────diff --git a/core/switch_core/db/stores/__init__.py b/core/switch_core/db/stores/__init__.pyindex 114b00a82..1ba7e80ba 100644--- a/core/switch_core/db/stores/__init__.py+++ b/core/switch_core/db/stores/__init__.py@@ -7,6 +7,7 @@ from switch_core.db.stores.document_store import DocumentStorefrom switch_core.db.stores.external_user_store import ExternalUserStorefrom switch_core.db.stores.invitation_store import InvitationStorefrom switch_core.db.stores.message_store import MessageStore+from switch_core.db.stores.messaging_event_store import MessagingEventReceiptStorefrom switch_core.db.stores.messaging_install_store import MessagingInstallStorefrom switch_core.db.stores.reference_store import ReferenceStorefrom switch_core.db.stores.reference_type_store import ReferenceTypeStore@@ -29,6 +30,7 @@ __all__ = ["ExternalUserStore","InvitationStore","MessageStore",+ "MessagingEventReceiptStore","MessagingInstallStore","ReferenceStore","ReferenceTypeStore",diff --git a/core/switch_core/db/stores/messaging_event_store.py b/core/switch_core/db/stores/messaging_event_store.pynew file mode 100644index 000000000..deea6a095--- /dev/null+++ b/core/switch_core/db/stores/messaging_event_store.py@@ -0,0 +1,94 @@+from __future__ import annotations++from datetime import UTC, datetime, timedelta++from sqlalchemy import delete, update+from sqlalchemy.exc import IntegrityError+from sqlalchemy.ext.asyncio import AsyncSession++from switch_core.db.models import MessagingEventReceipt++#: How long a handled event is remembered.+#:+#: Two different needs, and the longer one sets it. Deduplication needs only to+#: outlast the platform's retry schedule, which is tens of minutes — Slack has+#: given up long before an hour. `handled_at` is the reason for days: a claimed+#: receipt that never completed is an event that reached nobody, and an hour is+#: not long enough for anyone to notice and come looking. A week of one row per+#: inbound event is a table Postgres does not notice.+RECEIPT_RETENTION = timedelta(days=7)+++class MessagingEventReceiptStore:+ """Who has already taken an inbound event, and whether they finished it."""++ async def claim(+ self, session: AsyncSession, *, platform: str, external_event_id: str+ ) -> MessagingEventReceipt | None:+ """Take an event to handle, or return `None` because someone else has.++ The insert *is* the claim. Reading for an existing receipt and then+ writing one would leave the window this exists to close: two retries+ arriving together both read nothing, both write, and the customer gets+ two answers. Here the unique index decides, and the loser learns it did+ by failing.++ `None` rather than an exception because a duplicate is the ordinary+ case this is built for, not a fault — platforms deliver at least once+ by design, and a caller that treats the second one as an error would+ fill the log with the system working correctly.++ **Commit before doing the work.** An uncommitted claim still holds the+ index entry, so a concurrent retry's insert blocks on it rather than+ failing — and would stay blocked for as long as the first delivery+ takes, which is the length of an agent's turn. Committing straight away+ turns that wait into the immediate refusal it should be.+ """+ receipt = MessagingEventReceipt(+ platform=platform, external_event_id=external_event_id+ )+ session.add(receipt)+ try:+ await session.flush()+ except IntegrityError as exc:+ if "uq_messaging_event_receipts_event" not in str(exc.orig):+ raise+ await session.rollback()+ return None+ return receipt++ async def mark_handled(self, session: AsyncSession, *, receipt_id: str) -> None:+ """Record that the claimed event was seen all the way through.++ Its absence is the useful half. A receipt still unhandled long after it+ was claimed is an event the platform was told we had and nobody ever+ answered — a process that died mid-turn — and without this column that+ is indistinguishable from an event handled perfectly.+ """+ await session.execute(+ update(MessagingEventReceipt)+ .where(MessagingEventReceipt.id == receipt_id)+ .values(handled_at=datetime.now(UTC))+ )++ async def prune(self, session: AsyncSession) -> int:+ """Drop the bound tenant's receipts older than the retention window.++ Opportunistic rather than scheduled, the way stale role leases are+ cleared when the next agent takes a role. This backend has no janitor+ to hang a sweep on, and inventing one for a single table would be a+ larger change than the table deserves; running it off the traffic that+ creates the rows keeps the work proportional to that traffic and needs+ nothing started at boot.++ Called after the event has been dispatched, not before it. A prune in+ the claiming transaction would sit between a retry and the refusal it+ is waiting for.+ """+ result = await session.execute(+ delete(MessagingEventReceipt).where(+ MessagingEventReceipt.received_at+ < datetime.now(UTC) - RECEIPT_RETENTION+ )+ )+ return int(result.rowcount or 0) # type: ignore[attr-defined]diff --git a/core/switch_core/main.py b/core/switch_core/main.pyindex 067e995d5..12ce05586 100644--- a/core/switch_core/main.py+++ b/core/switch_core/main.py@@ -106,6 +106,7 @@ from switch_core.db.stores.external_user_store import ExternalUserStorefrom switch_core.db.stores.invitation_store import InvitationStorefrom switch_core.db.stores.media_store import MediaStorefrom switch_core.db.stores.message_store import MessageStore+from switch_core.db.stores.messaging_event_store import MessagingEventReceiptStorefrom switch_core.db.stores.messaging_install_store import MessagingInstallStorefrom switch_core.db.stores.package_store import PackageStorefrom switch_core.db.stores.reference_store import ReferenceStore@@ -506,6 +507,7 @@ async def run(config: SwitchConfig) -> None:install_service = MessagingInstallService(session_factory=session_factory,store=MessagingInstallStore(),+ receipts=MessagingEventReceiptStore(),installers=installers,lifecycle=collab_lifecycle,public_origin=config.messaging_public_url,diff --git a/core/switch_core/migrations/versions/b2e9d41c7f60_messaging_event_receipts.py b/core/switch_core/migrations/versions/b2e9d41c7f60_messaging_event_receipts.pynew file mode 100644index 000000000..89249574e--- /dev/null+++ b/core/switch_core/migrations/versions/b2e9d41c7f60_messaging_event_receipts.py@@ -0,0 +1,86 @@+"""messaging_event_receipts++A platform that retries is a platform that will eventually deliver the same+event twice, and the visible failure is an agent answering one question twice+in a customer's channel. This table is what makes the second delivery cheap to+recognise: a row is written before the work starts, the unique index arbitrates+between concurrent retries, and the loser stops.++`received_at` carries an index of its own because pruning reads that column and+nothing else. Receipts are only interesting for as long as the platform might+still re-send, and without a sweep the table would grow with every message the+busiest workspace ever sends.++The `tenant_isolation` policy is written out below rather than inherited,+because `265ed188ad6f` installed the policies as they stood then and a table+added afterwards has to bring its own. The DDL is a verbatim copy of+`switch_core/db/rls_ddl.py` as it stood when this migration was written,+copied rather than imported for the reason every revision in this chain+copies: a migration records a change that already happened, and importing the+live module would let a later edit silently change what this one means.+`tests/switch_core/db/test_frozen_ddl_matches_create_all.py` is what keeps the+copy from drifting.++Revision ID: b2e9d41c7f60+Revises: a7f2c3e9b481+Create Date: 2026-09-15 00:00:00.000000++"""++from collections.abc import Sequence++import sqlalchemy as sa+from alembic import op++revision: str = "b2e9d41c7f60"+down_revision: str | None = "a7f2c3e9b481"+branch_labels: str | Sequence[str] | None = None+depends_on: str | Sequence[str] | None = None++TABLE = "messaging_event_receipts"+POLICY_NAME = "tenant_isolation"++ENABLE_RLS = f'ALTER TABLE "{TABLE}" ENABLE ROW LEVEL SECURITY'++CREATE_POLICY = f"""CREATE POLICY {POLICY_NAME} ON "{TABLE}"+ FOR ALL+ USING ("tenant_id" = (SELECT require_tenant_id()))+ WITH CHECK ("tenant_id" = (SELECT require_tenant_id()))"""++DROP_POLICY = f'DROP POLICY IF EXISTS {POLICY_NAME} ON "{TABLE}"'+++def upgrade() -> None:+ op.create_table(+ TABLE,+ sa.Column("id", sa.Text(), nullable=False),+ sa.Column("tenant_id", sa.Text(), nullable=False),+ sa.Column("platform", sa.Text(), nullable=False),+ sa.Column("external_event_id", sa.Text(), nullable=False),+ sa.Column(+ "received_at",+ sa.DateTime(timezone=True),+ server_default=sa.text("now()"),+ nullable=False,+ ),+ sa.Column("handled_at", sa.DateTime(timezone=True), nullable=True),+ sa.PrimaryKeyConstraint("id"),+ sa.ForeignKeyConstraint(+ ["tenant_id"], ["tenants.id"], name="fk_messaging_event_receipts_tenant"+ ),+ sa.UniqueConstraint(+ "tenant_id",+ "platform",+ "external_event_id",+ name="uq_messaging_event_receipts_event",+ ),+ )+ op.create_index("ix_messaging_event_receipts_received_at", TABLE, ["received_at"])+ op.execute(ENABLE_RLS)+ op.execute(CREATE_POLICY)+++def downgrade() -> None:+ op.execute(DROP_POLICY)+ op.drop_index("ix_messaging_event_receipts_received_at", table_name=TABLE)+ op.drop_table(TABLE)diff --git a/core/tests/switch_core/bridges/collaboration/test_install_service.py b/core/tests/switch_core/bridges/collaboration/test_install_service.pyindex c7f0252f6..a6879881d 100644--- a/core/tests/switch_core/bridges/collaboration/test_install_service.py+++ b/core/tests/switch_core/bridges/collaboration/test_install_service.py@@ -47,6 +47,7 @@ from switch_core.db.models import (User,)from switch_core.db.session_scope import tenant_session+from switch_core.db.stores.messaging_event_store import MessagingEventReceiptStorefrom switch_core.db.stores.messaging_install_store import (INSTALL_ACTIVE,INSTALL_DISCONNECTED,@@ -104,9 +105,15 @@ class _FakeInstaller(MessagingAppInstaller):return Nonedef parse_webhook(- self, *, endpoint: WebhookEndpoint, body: bytes+ self, *, endpoint: WebhookEndpoint, headers: Mapping[str, str], body: bytes) -> InboundWebhook:- return InboundWebhook(envelope_type=endpoint, payload={}, handshake=None)+ return InboundWebhook(+ envelope_type=endpoint,+ payload={},+ handshake=None,+ external_event_id=None,+ delivery_attempt=0,+ )def workspace_of_event(self, payload: Mapping[str, object]) -> str:return self.workspace_id@@ -204,6 +211,7 @@ async def _fixture(harness: RLSHarness) -> _Fixture:fixture.service = MessagingInstallService(session_factory=harness.restricted,store=MessagingInstallStore(),+ receipts=MessagingEventReceiptStore(),installers=installers,lifecycle=fixture.lifecycle, # type: ignore[arg-type]public_origin=_ORIGIN,diff --git a/core/tests/switch_core/bridges/collaboration/test_install_webhook.py b/core/tests/switch_core/bridges/collaboration/test_install_webhook.pyindex fd4244e92..7b72dc33f 100644--- a/core/tests/switch_core/bridges/collaboration/test_install_webhook.py+++ b/core/tests/switch_core/bridges/collaboration/test_install_webhook.py@@ -19,18 +19,21 @@ socket: the bridge lifecycle and the adapter at the end of it.from __future__ import annotations+import asyncioimport hashlibimport hmacimport jsonimport timeimport uuidfrom dataclasses import dataclass+from datetime import UTC, datetime, timedeltafrom typing import Anyfrom urllib.parse import urlencodeimport httpximport pytestfrom fastapi import FastAPI+from sqlalchemy import selectfrom sqlalchemy.ext.asyncio import async_sessionmakerfrom switch_core.bridges.collaboration.adapter import CollaborationAdapter@@ -48,11 +51,16 @@ from switch_core.bridges.collaboration.slack.install import SlackAppInstallerfrom switch_core.db.models import (Client,CollaborationBridge,+ MessagingEventReceipt,MessagingInstall,Tenant,User,)from switch_core.db.session_scope import tenant_session+from switch_core.db.stores.messaging_event_store import (+ RECEIPT_RETENTION,+ MessagingEventReceiptStore,+)from switch_core.db.stores.messaging_install_store import MessagingInstallStorefrom switch_core.tenant_context import current_tenant_idfrom tests.conftest import RLSHarness@@ -125,6 +133,29 @@ class _RecordingAdapter(_SocketOnlyAdapter):self.tenants_bound.append(current_tenant_id())+class _GatedAdapter(_SocketOnlyAdapter):+ """An adapter that holds a dispatch open until it is let go.++ What it buys is determinism. The race worth testing is a retry arriving+ while the first delivery is still working — the whole reason the claim is+ written before the dispatch rather than after — and an adapter that returns+ instantly would let the first delivery finish before the second began,+ proving only that a repeat is refused once the first is over.+ """++ def __init__(self) -> None:+ self.dispatched: list[tuple[str, dict[str, Any]]] = []+ self.entered = asyncio.Event()+ self.release = asyncio.Event()++ async def dispatch_event(+ self, *, envelope_type: str, payload: dict[str, Any]+ ) -> None:+ self.dispatched.append((envelope_type, payload))+ self.entered.set()+ await self.release.wait()++class _FakeLifecycle:def __init__(self, factory: async_sessionmaker) -> None:self._factory = factory@@ -247,6 +278,7 @@ async def _fixture(harness: RLSHarness) -> _Fixture:fixture.service = MessagingInstallService(session_factory=harness.restricted,store=MessagingInstallStore(),+ receipts=MessagingEventReceiptStore(),installers=installers,lifecycle=fixture.lifecycle, # type: ignore[arg-type]public_origin=_ORIGIN,@@ -271,6 +303,30 @@ def _event(workspace_id: str, text: str) -> bytes:).encode()+def _numbered_event(workspace_id: str, text: str, event_id: str) -> bytes:+ """An event carrying the id Slack puts on everything it retries.++ Separate from `_event` rather than an argument to it, so the tests above go+ on exercising the unnumbered path — which is the one a slash command and an+ interaction take, and which must dispatch rather than being refused for+ having nothing to deduplicate by.+ """+ return json.dumps(+ {+ "type": "event_callback",+ "team_id": workspace_id,+ "event_id": event_id,+ "event": {"type": "message", "text": text, "channel": "C1"},+ }+ ).encode()+++async def _receipts(factory: async_sessionmaker, tenant_id: str) -> list[Any]:+ async with tenant_session(factory, tenant_id) as session:+ rows = await session.execute(select(MessagingEventReceipt))+ return list(rows.scalars())++def _uninstalled(workspace_id: str) -> bytes:return json.dumps({@@ -567,6 +623,197 @@ class TestABridgeThatCannotTakeEvents:await fixture.service.deliver(target, event)+class TestAnEventIsHandledOnce:+ """The platform delivers at least once; the customer must hear once.++ Slack allows three seconds to acknowledge and re-sends what it does not get+ an answer to, so a deployment under load is told the same thing twice. The+ payload of a retry is byte-identical to the original — the id is the only+ thing that distinguishes it from someone saying the same words again — and+ the visible failure is an agent answering one question twice in a+ customer's channel.+ """++ async def test_a_retry_of_the_same_event_dispatches_once(+ self, rls_harness: RLSHarness+ ) -> None:+ fixture = await _fixture(rls_harness)+ body = _numbered_event(fixture.a.workspace_id, "hello", "Ev1")++ first = await _post(fixture, events_path("slack"), body)+ second = await _post(fixture, events_path("slack"), body)++ assert (first.status_code, second.status_code) == (200, 200)+ assert _texts(fixture.a.adapter) == ["hello"]++ async def test_a_retry_arriving_mid_turn_loses_to_the_delivery_in_flight(+ self, rls_harness: RLSHarness+ ) -> None:+ """The case the ordering exists for, and the only one that is a race.++ Recording the receipt after the work instead of before would order the+ two the wrong way round: both would find nothing claimed, both would+ dispatch, and the duplicate would be noticed once it no longer+ mattered.+ """+ fixture = await _fixture(rls_harness)+ gated = _GatedAdapter()+ fixture.lifecycle.adapters[fixture.a.bridge_id] = gated+ body = _numbered_event(fixture.a.workspace_id, "hello", "Ev1")+ event = fixture.service.authenticate(+ platform="slack", endpoint="events", headers=_signed(body), body=body+ )+ target = await fixture.service.resolve(platform="slack", event=event)++ in_flight = asyncio.create_task(fixture.service.deliver(target, event))+ await gated.entered.wait()+ await fixture.service.deliver(target, event)++ assert len(gated.dispatched) == 1+ gated.release.set()+ await in_flight++ async def test_an_event_with_no_id_is_dispatched_every_time(+ self, rls_harness: RLSHarness+ ) -> None:+ """Not a weaker guarantee quietly accepted.++ Slack numbers only the envelopes it retries, so one arriving without an+ id arrives exactly once. Two of them are two real messages, and+ refusing the second for having nothing to deduplicate by would drop a+ customer's message on the floor.+ """+ fixture = await _fixture(rls_harness)+ body = _event(fixture.a.workspace_id, "same words")++ await _post(fixture, events_path("slack"), body)+ await _post(fixture, events_path("slack"), body)++ assert _texts(fixture.a.adapter) == ["same words", "same words"]+ assert await _receipts(rls_harness.restricted, fixture.a.tenant_id) == []++ async def test_one_tenants_event_ids_do_not_block_anothers(+ self, rls_harness: RLSHarness+ ) -> None:+ """Uniqueness is per tenant, and this is why that is the shape to want.++ Slack's ids are unique in its own namespace, so a deployment-wide index+ would protect exactly as well — but it would let one customer's traffic+ refuse another's, and a conflict would name a row the inserting tenant+ cannot see.+ """+ fixture = await _fixture(rls_harness)++ await _post(+ fixture,+ events_path("slack"),+ _numbered_event(fixture.a.workspace_id, "a", "Ev-shared"),+ )+ await _post(+ fixture,+ events_path("slack"),+ _numbered_event(fixture.b.workspace_id, "b", "Ev-shared"),+ )++ assert _texts(fixture.a.adapter) == ["a"]+ assert _texts(fixture.b.adapter) == ["b"]++ async def test_a_completed_delivery_is_recorded_as_handled(+ self, rls_harness: RLSHarness+ ) -> None:+ """The absence of this is the useful half.++ Claiming before the work chooses at-most-once: an event taken by a+ process that then dies is not retried, because the platform has already+ been told 200. `handled_at` is what keeps that loss findable — a+ claimed receipt that never completed is a real event that reached+ nobody.+ """+ fixture = await _fixture(rls_harness)++ await _post(+ fixture,+ events_path("slack"),+ _numbered_event(fixture.a.workspace_id, "hello", "Ev1"),+ )++ rows = await _receipts(rls_harness.restricted, fixture.a.tenant_id)+ assert [(row.platform, row.external_event_id) for row in rows] == [+ ("slack", "Ev1")+ ]+ assert rows[0].handled_at is not None++ async def test_a_receipt_past_its_retention_is_pruned_by_the_next_event(+ self, rls_harness: RLSHarness+ ) -> None:+ """Opportunistic, because this backend has no janitor to hang it on.++ The table would otherwise grow with every message the busiest workspace+ ever sends. Running the sweep off the traffic that creates the rows+ keeps the work proportional to that traffic and needs nothing started+ at boot.+ """+ fixture = await _fixture(rls_harness)+ async with tenant_session(+ rls_harness.restricted, fixture.a.tenant_id+ ) as session:+ session.add(+ MessagingEventReceipt(+ tenant_id=fixture.a.tenant_id,+ platform="slack",+ external_event_id="Ev-ancient",+ received_at=datetime.now(UTC)+ - RECEIPT_RETENTION+ - timedelta(days=1),+ )+ )+ await session.commit()++ await _post(+ fixture,+ events_path("slack"),+ _numbered_event(fixture.a.workspace_id, "hello", "Ev-fresh"),+ )++ rows = await _receipts(rls_harness.restricted, fixture.a.tenant_id)+ assert [row.external_event_id for row in rows] == ["Ev-fresh"]++ async def test_a_retry_of_an_event_nobody_holds_is_still_dropped_at_resolve(+ self, rls_harness: RLSHarness+ ) -> None:+ """Deduplication sits after resolution, so an unknown workspace writes+ no receipt at all — there is no tenant to write it as."""+ fixture = await _fixture(rls_harness)++ response = await _post(+ fixture, events_path("slack"), _numbered_event("T-nobody", "x", "Ev1")+ )++ assert response.status_code == 200+ assert await _receipts(rls_harness.restricted, fixture.a.tenant_id) == []++ async def test_a_bridge_that_was_down_still_takes_the_retry(+ self, rls_harness: RLSHarness+ ) -> None:+ """The 503 and the receipt compose, and this is the case that proves it.++ A bridge mid-restart makes the event fail before any claim is written,+ so the retry Slack sends in response finds nothing taken and is handled+ normally. Claiming any earlier — before `resolve` — would turn a+ transient outage into a permanently lost message.+ """+ fixture = await _fixture(rls_harness)+ body = _numbered_event(fixture.a.workspace_id, "hello", "Ev1")+ del fixture.lifecycle.adapters[fixture.a.bridge_id]++ refused = await _post(fixture, events_path("slack"), body)+ fixture.lifecycle.adapters[fixture.a.bridge_id] = fixture.a.adapter+ retried = await _post(fixture, events_path("slack"), body)++ assert (refused.status_code, retried.status_code) == (503, 200)+ assert _texts(fixture.a.adapter) == ["hello"]++class TestThePlatformSayingTheInstallIsOver:"""`app_uninstalled` arrives on the same URL as everything else."""diff --git a/core/tests/switch_core/bridges/collaboration/test_slack_installer.py b/core/tests/switch_core/bridges/collaboration/test_slack_installer.pyindex 5c7cb4c47..ea8695a44 100644--- a/core/tests/switch_core/bridges/collaboration/test_slack_installer.py+++ b/core/tests/switch_core/bridges/collaboration/test_slack_installer.py@@ -21,6 +21,7 @@ from switch_core.bridges.collaboration.install import (MessagingInstallerRegistry,MessagingInstallError,WebhookAuthenticityError,+ WebhookEndpoint,WebhookPayloadError,events_path,oauth_callback_path,@@ -234,7 +235,7 @@ class TestParsingAWebhook:}).encode()- parsed = installer.parse_webhook(endpoint="events", body=body)+ parsed = installer.parse_webhook(endpoint="events", headers={}, body=body)assert parsed.envelope_type == "events_api"assert parsed.handshake is None@@ -254,7 +255,7 @@ class TestParsingAWebhook:"""body = json.dumps({"type": "url_verification", "challenge": "abc123"}).encode()- parsed = installer.parse_webhook(endpoint="events", body=body)+ parsed = installer.parse_webhook(endpoint="events", headers={}, body=body)assert parsed.handshake == "abc123"@@ -263,7 +264,7 @@ class TestParsingAWebhook:) -> None:with pytest.raises(WebhookPayloadError, match="challenge"):installer.parse_webhook(- endpoint="events", body=b'{"type":"url_verification"}'+ endpoint="events", headers={}, body=b'{"type":"url_verification"}')def test_a_slash_command_is_its_form_fields(@@ -273,7 +274,7 @@ class TestParsingAWebhook:{"command": "/agents-status", "text": "", "team_id": "T1", "user_id": "U1"}).encode()- parsed = installer.parse_webhook(endpoint="commands", body=body)+ parsed = installer.parse_webhook(endpoint="commands", headers={}, body=body)assert parsed.envelope_type == "slash_commands"assert parsed.payload["command"] == "/agents-status"@@ -287,7 +288,7 @@ class TestParsingAWebhook:inner = {"type": "block_actions", "team": {"id": "T1"}}body = urlencode({"payload": json.dumps(inner)}).encode()- parsed = installer.parse_webhook(endpoint="interactive", body=body)+ parsed = installer.parse_webhook(endpoint="interactive", headers={}, body=body)assert parsed.envelope_type == "interactive"assert parsed.payload == inner@@ -296,7 +297,106 @@ class TestParsingAWebhook:self, installer: SlackAppInstaller) -> None:with pytest.raises(WebhookPayloadError, match="payload"):- installer.parse_webhook(endpoint="interactive", body=b"other=1")+ installer.parse_webhook(endpoint="interactive", headers={}, body=b"other=1")++ def test_an_event_callback_carries_slacks_own_event_id(+ self, installer: SlackAppInstaller+ ) -> None:+ """The id is the only thing two copies of one event have in common.++ The payload of a retry is byte-identical to the original, so without+ this nothing downstream could tell a re-send from somebody saying the+ same words twice.+ """+ body = json.dumps(+ {"type": "event_callback", "team_id": "T1", "event_id": "Ev123"}+ ).encode()++ parsed = installer.parse_webhook(endpoint="events", headers={}, body=body)++ assert parsed.external_event_id == "Ev123"++ @pytest.mark.parametrize(+ "envelope",+ [+ pytest.param({"type": "event_callback", "team_id": "T1"}, id="absent"),+ pytest.param(+ {"type": "event_callback", "team_id": "T1", "event_id": ""},+ id="empty",+ ),+ pytest.param(+ {"type": "event_callback", "team_id": "T1", "event_id": 7},+ id="not-a-string",+ ),+ ],+ )+ def test_an_event_callback_with_no_usable_id_deduplicates_by_nothing(+ self, installer: SlackAppInstaller, envelope: dict[str, object]+ ) -> None:+ """No id means dispatch it, never refuse it.++ An empty or missing `event_id` is not a reason to drop a real message+ from a customer's channel — it only means there is nothing to key a+ receipt on, so the event is handled the way an unnumbered one is.+ """+ parsed = installer.parse_webhook(+ endpoint="events", headers={}, body=json.dumps(envelope).encode()+ )++ assert parsed.external_event_id is None++ @pytest.mark.parametrize(+ ("endpoint", "body"),+ [+ pytest.param(+ "commands", urlencode({"team_id": "T1"}).encode(), id="command"+ ),+ pytest.param(+ "interactive",+ urlencode({"payload": json.dumps({"team": {"id": "T1"}})}).encode(),+ id="interaction",+ ),+ ],+ )+ def test_the_form_endpoints_carry_no_id_because_slack_sends_them_once(+ self, installer: SlackAppInstaller, endpoint: WebhookEndpoint, body: bytes+ ) -> None:+ """Slack retries neither, so there is nothing to deduplicate.++ A command and an interaction are a person waiting on a dialog. Slack+ sends each exactly once and puts no id on it.+ """+ parsed = installer.parse_webhook(endpoint=endpoint, headers={}, body=body)++ assert parsed.external_event_id is None++ @pytest.mark.parametrize(+ ("headers", "expected"),+ [+ pytest.param({}, 0, id="first-delivery"),+ pytest.param({"x-slack-retry-num": "2"}, 2, id="second-retry"),+ pytest.param({"x-slack-retry-num": "nonsense"}, 0, id="unparseable"),+ ],+ )+ def test_the_retry_count_is_read_as_a_hint_and_never_as_a_decision(+ self,+ installer: SlackAppInstaller,+ headers: dict[str, str],+ expected: int,+ ) -> None:+ """Unsigned input, so a bad value must not be able to refuse an event.++ The retry header is outside the signature — which covers the body and+ the timestamp — so a garbled or forged one degrades to zero rather than+ raising. The worst a forgery achieves is a wrong number in a log line.+ """+ body = json.dumps(+ {"type": "event_callback", "team_id": "T1", "event_id": "Ev1"}+ ).encode()++ parsed = installer.parse_webhook(endpoint="events", headers=headers, body=body)++ assert parsed.delivery_attempt == expected@pytest.mark.parametrize("body",@@ -316,7 +416,7 @@ class TestParsingAWebhook:rather than a shrug."""with pytest.raises(WebhookPayloadError):- installer.parse_webhook(endpoint="events", body=body)+ installer.parse_webhook(endpoint="events", headers={}, body=body)class TestWhichWorkspaceSentIt: