Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

feat(messaging): handle an inbound platform event once (CHOO-2626)

Petr Bauch9fcffc32fb0d
diff --git a/core/switch_core/bridges/collaboration/install.py b/core/switch_core/bridges/collaboration/install.py
index aa1b2d6fa..ceb5c16a8 100644
--- a/core/switch_core/bridges/collaboration/install.py
+++ b/core/switch_core/bridges/collaboration/install.py
@@ -139,11 +139,26 @@ class InboundWebhook:
is `None` for every real event, and it arrives before any workspace has
installed anything — so it must be answerable with no tenant, no install
row, and nothing running.
+
+ `external_event_id` is the platform's own id for this delivery, and the
+ only thing two copies of one event have in common — the payload is
+ identical, so nothing else could tell a retry from a second message saying
+ the same words. `None` means the platform does not number this kind of
+ envelope, which on Slack means the kind it also does not retry; it never
+ means "this one was not checked".
+
+ `delivery_attempt` is how many times the platform has given up on us and
+ sent this again, zero on a first delivery. It changes nothing about how the
+ event is handled — the receipt decides that — and is carried because it is
+ the only place the deployment is told its own acknowledgements are arriving
+ too late.
"""
envelope_type: str
payload: dict[str, Any]
handshake: str | None
+ external_event_id: str | None
+ delivery_attempt: int
@dataclass(frozen=True)
@@ -245,9 +260,9 @@ class MessagingAppInstaller(ABC):
@abstractmethod
def parse_webhook(
- self, *, endpoint: WebhookEndpoint, body: bytes
+ self, *, endpoint: WebhookEndpoint, headers: Mapping[str, str], body: bytes
) -> InboundWebhook:
- """Read a verified request body into an event a running adapter takes.
+ """Read a verified request into an event a running adapter takes.
Called only after :meth:`verify_webhook` has passed, and separate from
it for exactly that reason: parsing before verifying is how an
@@ -257,6 +272,11 @@ class MessagingAppInstaller(ABC):
method knows the encoding, which is per platform and per endpoint —
Slack posts JSON to one of its three and form data to the other two.
+ Takes the headers as well as the body because a delivery is described
+ in both: the event is in the body, and how many times it has been sent
+ is in a header. Which header, and whether there is one, is the
+ platform's business rather than the route's.
+
Raise :class:`WebhookPayloadError` for a body that cannot be read.
"""
diff --git a/core/switch_core/bridges/collaboration/install_routes.py b/core/switch_core/bridges/collaboration/install_routes.py
index 899ec55fe..fb8818228 100644
--- a/core/switch_core/bridges/collaboration/install_routes.py
+++ b/core/switch_core/bridges/collaboration/install_routes.py
@@ -240,6 +240,20 @@ def create_messaging_install_router(
logger.info("Answered a %s URL verification", platform)
return PlainTextResponse(event.handshake)
+ if event.delivery_attempt > 0:
+ # The only signal this deployment gets that its own acknowledgements
+ # are arriving too late. The event itself is handled normally — the
+ # receipt decides whether it is a duplicate — but a run of these is
+ # the platform saying the three-second answer is being missed, and
+ # nothing else in the system would say so.
+ logger.warning(
+ "%s is re-sending a %s event (attempt %s), which means an earlier "
+ "delivery was not acknowledged in time",
+ platform,
+ event.envelope_type,
+ event.delivery_attempt,
+ )
+
try:
# Before resolving, because this is the one event that arrives as
# the bridge it would be resolved to is going away.
diff --git a/core/switch_core/bridges/collaboration/install_service.py b/core/switch_core/bridges/collaboration/install_service.py
index bacbb1fbe..062d3f678 100644
--- a/core/switch_core/bridges/collaboration/install_service.py
+++ b/core/switch_core/bridges/collaboration/install_service.py
@@ -77,6 +77,7 @@ from switch_core.bridges.collaboration.lifecycle_service import (
from switch_core.crypto import decrypt_token, encrypt_token
from switch_core.db.models import MessagingInstall
from switch_core.db.session_scope import tenant_session
+from switch_core.db.stores.messaging_event_store import MessagingEventReceiptStore
from switch_core.db.stores.messaging_install_store import (
INSTALL_ACTIVE,
INSTALL_DISCONNECTED,
@@ -120,9 +121,15 @@ class Revocation:
@dataclass(frozen=True)
class WebhookTarget:
- """Where one verified event goes: a tenant, a bridge, and its live adapter."""
+ """Where one verified event goes: a tenant, a bridge, and its live adapter.
+
+ `platform` is carried rather than passed alongside because it is part of
+ the answer: the same workspace id could in principle be issued by two
+ platforms, and every row written about this delivery is keyed by the pair.
+ """
tenant_id: str
+ platform: str
bridge_id: str
adapter: CollaborationAdapter
@@ -144,6 +151,7 @@ class MessagingInstallService:
*,
session_factory: async_sessionmaker[AsyncSession],
store: MessagingInstallStore,
+ receipts: MessagingEventReceiptStore,
installers: MessagingInstallerRegistry,
lifecycle: CollaborationBridgeLifecycleService,
public_origin: str,
@@ -151,6 +159,7 @@ class MessagingInstallService:
) -> None:
self._session_factory = session_factory
self._store = store
+ self._receipts = receipts
self._installers = installers
self._lifecycle = lifecycle
self._public_origin = public_origin
@@ -426,7 +435,7 @@ class MessagingInstallService:
"""
installer = self._installers.get(platform)
installer.verify_webhook(headers=headers, body=body)
- return installer.parse_webhook(endpoint=endpoint, body=body)
+ return installer.parse_webhook(endpoint=endpoint, headers=headers, body=body)
def revocation(self, *, platform: str, event: InboundWebhook) -> Revocation | None:
"""Whether this event is the platform ending the install, and for whom.
@@ -492,20 +501,75 @@ class MessagingInstallService:
f"{workspace_id}, is not running"
)
return WebhookTarget(
- tenant_id=tenant_id, bridge_id=install.bridge_id, adapter=adapter
+ tenant_id=tenant_id,
+ platform=platform,
+ bridge_id=install.bridge_id,
+ adapter=adapter,
)
async def deliver(self, target: WebhookTarget, event: InboundWebhook) -> None:
- """Hand a resolved event to the bridge, as its own transport would.
-
- With **nothing bound**, which is not an oversight. A bridge that
- receives over a socket dispatches from a task that binds no tenant, and
- every handler below it binds the tenant of the room it is acting on.
- Binding here would make the two delivery paths differ in the one
- respect that decides who a message reaches, and would hide a handler
- that had forgotten to bind for itself — for exactly as long as it took
- someone to receive the same event over a socket instead.
+ """Hand a resolved event to the bridge, at most once.
+
+ Claiming comes first and the dispatch second, so a retry that arrives
+ while the first delivery is still working finds the event taken and
+ stops. That ordering is the whole of the deduplication: the platform
+ sends the same event again whenever it is not answered in time, and
+ the payload of a retry is identical to the original, so nothing later
+ in the stack could tell it from someone saying the same thing twice.
+
+ An event the platform does not number is dispatched without a claim.
+ That is not a weaker guarantee quietly accepted — Slack numbers what it
+ retries, so an envelope with no id is one that arrives exactly once.
+
+ The dispatch itself runs with **nothing bound**, which is not an
+ oversight. A bridge that receives over a socket dispatches from a task
+ that binds no tenant, and every handler below it binds the tenant of
+ the room it is acting on. Binding here would make the two delivery
+ paths differ in the one respect that decides who a message reaches, and
+ would hide a handler that had forgotten to bind for itself — for
+ exactly as long as it took someone to receive the same event over a
+ socket instead.
"""
+ if event.external_event_id is None:
+ await self._dispatch(target, event)
+ return
+
+ async with tenant_session(self._session_factory, target.tenant_id) as session:
+ receipt = await self._receipts.claim(
+ session,
+ platform=target.platform,
+ external_event_id=event.external_event_id,
+ )
+ if receipt is None:
+ logger.info(
+ "Dropped a repeat delivery of %s event %s to bridge %s "
+ "(attempt %s); it has already been taken",
+ target.platform,
+ event.external_event_id,
+ target.bridge_id,
+ event.delivery_attempt,
+ )
+ return
+ receipt_id = receipt.id
+ # Before the dispatch, not with it. The index entry this writes is
+ # what a concurrent retry collides with, and an uncommitted one
+ # makes that retry wait for the turn instead of losing to it.
+ await session.commit()
+
+ await self._dispatch(target, event)
+
+ async with tenant_session(self._session_factory, target.tenant_id) as session:
+ await self._receipts.mark_handled(session, receipt_id=receipt_id)
+ pruned = await self._receipts.prune(session)
+ await session.commit()
+ if pruned:
+ logger.info(
+ "Pruned %s expired messaging event receipts for tenant %s",
+ pruned,
+ target.tenant_id,
+ )
+
+ async def _dispatch(self, target: WebhookTarget, event: InboundWebhook) -> None:
with no_tenant():
await target.adapter.dispatch_event(
envelope_type=event.envelope_type, payload=event.payload
diff --git a/core/switch_core/bridges/collaboration/slack/install.py b/core/switch_core/bridges/collaboration/slack/install.py
index 848aada72..09beb5119 100644
--- a/core/switch_core/bridges/collaboration/slack/install.py
+++ b/core/switch_core/bridges/collaboration/slack/install.py
@@ -65,6 +65,30 @@ def _json_object(raw: bytes) -> dict[str, Any]:
return parsed
+#: Slack's count of how many times it has re-sent a delivery, absent on the
+#: first. Header names arrive from Starlette lower-cased and are compared that
+#: way; Slack's own spelling is `X-Slack-Retry-Num`.
+_RETRY_NUM_HEADER = "x-slack-retry-num"
+
+
+def _retry_number(headers: Mapping[str, str]) -> int:
+ """How many times Slack has sent this already, defaulting to none.
+
+ Unauthenticated in the sense that it is not covered by the signature — the
+ signature is over the body and the timestamp — so it is read as a hint and
+ never as a decision. A forged value cannot make an event be handled twice
+ or dropped, because the receipt decides that; the worst it can do is put a
+ wrong number in a log line.
+ """
+ raw = headers.get(_RETRY_NUM_HEADER)
+ if raw is None:
+ return 0
+ try:
+ return int(raw)
+ except ValueError:
+ return 0
+
+
def _form_fields(raw: bytes) -> dict[str, Any]:
"""Slack's form encoding as a flat dict, matching Socket Mode's payload.
@@ -215,8 +239,10 @@ class SlackAppInstaller(MessagingAppInstaller):
raise WebhookAuthenticityError("bad Slack signature")
def parse_webhook(
- self, *, endpoint: WebhookEndpoint, body: bytes
+ self, *, endpoint: WebhookEndpoint, headers: Mapping[str, str], body: bytes
) -> InboundWebhook:
+ attempt = _retry_number(headers)
+
if endpoint == "commands":
# A slash command posts its fields as a form, and Socket Mode
# delivers that same flat dict — so the parsed form *is* the
@@ -225,6 +251,13 @@ class SlackAppInstaller(MessagingAppInstaller):
envelope_type="slash_commands",
payload=_form_fields(body),
handshake=None,
+ # Slack retries neither of the two form endpoints. A command
+ # and an interaction are a person waiting on a dialog, and a
+ # reply that arrives a minute late is worse than none — so
+ # Slack sends each exactly once and there is no id on it to
+ # deduplicate by.
+ external_event_id=None,
+ delivery_attempt=attempt,
)
if endpoint == "interactive":
@@ -239,6 +272,8 @@ class SlackAppInstaller(MessagingAppInstaller):
envelope_type="interactive",
payload=_json_object(raw.encode()),
handshake=None,
+ external_event_id=None,
+ delivery_attempt=attempt,
)
envelope = _json_object(body)
@@ -253,11 +288,25 @@ class SlackAppInstaller(MessagingAppInstaller):
"Slack sent a URL verification with no challenge to echo"
)
return InboundWebhook(
- envelope_type="url_verification", payload=envelope, handshake=challenge
+ envelope_type="url_verification",
+ payload=envelope,
+ handshake=challenge,
+ external_event_id=None,
+ delivery_attempt=attempt,
)
+ event_id = envelope.get("event_id")
return InboundWebhook(
- envelope_type="events_api", payload=envelope, handshake=None
+ envelope_type="events_api",
+ payload=envelope,
+ handshake=None,
+ # The one envelope Slack numbers, and the one it retries. A
+ # retried delivery carries the same `event_id` as the original,
+ # which is the whole basis of handling it once.
+ external_event_id=event_id
+ if isinstance(event_id, str) and event_id
+ else None,
+ delivery_attempt=attempt,
)
def workspace_of_event(self, payload: Mapping[str, object]) -> str:
diff --git a/core/switch_core/db/models.py b/core/switch_core/db/models.py
index 2099d6ed6..caa5dc649 100644
--- a/core/switch_core/db/models.py
+++ b/core/switch_core/db/models.py
@@ -1280,6 +1280,72 @@ class MessagingInstallState(TenantScoped, Base):
)
+class MessagingEventReceipt(TenantScoped, Base):
+ """One inbound platform event, claimed once so it is handled once.
+
+ Platforms deliver at least once. Slack gives three seconds to acknowledge
+ an event and retries what it does not get an answer to — so a deployment
+ under load, or one restarted mid-request, is told the same thing again.
+ Without a record of what has been taken, the second telling produces a
+ second answer in the customer's channel, which is the visible failure: an
+ agent replying twice to one question.
+
+ **The row is written before the work, not after**, and the unique index is
+ what arbitrates. Two retries in flight at once both reach the insert and
+ exactly one survives it; the loser stops there. Recording afterwards would
+ order the two the wrong way round — both would dispatch, and the duplicate
+ would be detected once it no longer mattered.
+
+ That ordering chooses at-most-once over at-least-once, which is worth
+ stating plainly: an event claimed by a process that then dies is not
+ retried, because the platform has already been told 200 and this table says
+ the event is taken. It is not a new loss. The route has acknowledged before
+ handling since it was written — it has to, the deadline is shorter than a
+ turn — so the event was already unrecoverable at that point. What this adds
+ is `handled_at`, which makes the loss visible: a claimed row that never
+ completed is a real event that reached nobody, and it can be found.
+
+ `external_event_id` is the platform's own id for the delivery, and only
+ some envelopes have one. Slack numbers Events API envelopes and retries
+ only those; a slash command and an interaction get one shot and no id, so
+ there is nothing to deduplicate and no row here. A missing id means "the
+ platform does not retry this", not "this was not checked".
+
+ Uniqueness is `(tenant_id, platform, external_event_id)` and not the
+ deployment-wide pair, unlike the workspace claim on `messaging_installs`.
+ The two would be equivalent — an event id is unique in the platform's own
+ namespace and a workspace belongs to one tenant — so the tenant-local index
+ is the one to prefer: it keeps one customer's event ids out of another's
+ namespace entirely, and it means a conflict is always with a row the
+ inserting tenant can actually see rather than an opaque refusal naming
+ somebody else's.
+ """
+
+ __tablename__ = "messaging_event_receipts"
+ __table_args__ = (
+ UniqueConstraint(
+ "tenant_id",
+ "platform",
+ "external_event_id",
+ name="uq_messaging_event_receipts_event",
+ ),
+ # Pruning reads this and nothing else. Receipts are only useful for as
+ # long as the platform might still retry, and the table would otherwise
+ # grow with every message the busiest workspace ever sends.
+ Index("ix_messaging_event_receipts_received_at", "received_at"),
+ )
+
+ id: Mapped[str] = mapped_column(Text, primary_key=True, default=_uuid)
+ platform: Mapped[str] = mapped_column(Text, nullable=False)
+ external_event_id: Mapped[str] = mapped_column(Text, nullable=False)
+ received_at: Mapped[datetime] = mapped_column(
+ DateTime(timezone=True), server_default=func.now(), nullable=False
+ )
+ handled_at: Mapped[datetime | None] = mapped_column(
+ DateTime(timezone=True), nullable=True
+ )
+
+
# ── Server-Side Connectors ────────────────────────────────────────────────────
diff --git a/core/switch_core/db/stores/__init__.py b/core/switch_core/db/stores/__init__.py
index 114b00a82..1ba7e80ba 100644
--- a/core/switch_core/db/stores/__init__.py
+++ b/core/switch_core/db/stores/__init__.py
@@ -7,6 +7,7 @@ from switch_core.db.stores.document_store import DocumentStore
from switch_core.db.stores.external_user_store import ExternalUserStore
from switch_core.db.stores.invitation_store import InvitationStore
from switch_core.db.stores.message_store import MessageStore
+from switch_core.db.stores.messaging_event_store import MessagingEventReceiptStore
from switch_core.db.stores.messaging_install_store import MessagingInstallStore
from switch_core.db.stores.reference_store import ReferenceStore
from switch_core.db.stores.reference_type_store import ReferenceTypeStore
@@ -29,6 +30,7 @@ __all__ = [
"ExternalUserStore",
"InvitationStore",
"MessageStore",
+ "MessagingEventReceiptStore",
"MessagingInstallStore",
"ReferenceStore",
"ReferenceTypeStore",
diff --git a/core/switch_core/db/stores/messaging_event_store.py b/core/switch_core/db/stores/messaging_event_store.py
new file mode 100644
index 000000000..deea6a095
--- /dev/null
+++ b/core/switch_core/db/stores/messaging_event_store.py
@@ -0,0 +1,94 @@
+from __future__ import annotations
+
+from datetime import UTC, datetime, timedelta
+
+from sqlalchemy import delete, update
+from sqlalchemy.exc import IntegrityError
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from switch_core.db.models import MessagingEventReceipt
+
+#: How long a handled event is remembered.
+#:
+#: Two different needs, and the longer one sets it. Deduplication needs only to
+#: outlast the platform's retry schedule, which is tens of minutes — Slack has
+#: given up long before an hour. `handled_at` is the reason for days: a claimed
+#: receipt that never completed is an event that reached nobody, and an hour is
+#: not long enough for anyone to notice and come looking. A week of one row per
+#: inbound event is a table Postgres does not notice.
+RECEIPT_RETENTION = timedelta(days=7)
+
+
+class MessagingEventReceiptStore:
+ """Who has already taken an inbound event, and whether they finished it."""
+
+ async def claim(
+ self, session: AsyncSession, *, platform: str, external_event_id: str
+ ) -> MessagingEventReceipt | None:
+ """Take an event to handle, or return `None` because someone else has.
+
+ The insert *is* the claim. Reading for an existing receipt and then
+ writing one would leave the window this exists to close: two retries
+ arriving together both read nothing, both write, and the customer gets
+ two answers. Here the unique index decides, and the loser learns it did
+ by failing.
+
+ `None` rather than an exception because a duplicate is the ordinary
+ case this is built for, not a fault — platforms deliver at least once
+ by design, and a caller that treats the second one as an error would
+ fill the log with the system working correctly.
+
+ **Commit before doing the work.** An uncommitted claim still holds the
+ index entry, so a concurrent retry's insert blocks on it rather than
+ failing — and would stay blocked for as long as the first delivery
+ takes, which is the length of an agent's turn. Committing straight away
+ turns that wait into the immediate refusal it should be.
+ """
+ receipt = MessagingEventReceipt(
+ platform=platform, external_event_id=external_event_id
+ )
+ session.add(receipt)
+ try:
+ await session.flush()
+ except IntegrityError as exc:
+ if "uq_messaging_event_receipts_event" not in str(exc.orig):
+ raise
+ await session.rollback()
+ return None
+ return receipt
+
+ async def mark_handled(self, session: AsyncSession, *, receipt_id: str) -> None:
+ """Record that the claimed event was seen all the way through.
+
+ Its absence is the useful half. A receipt still unhandled long after it
+ was claimed is an event the platform was told we had and nobody ever
+ answered — a process that died mid-turn — and without this column that
+ is indistinguishable from an event handled perfectly.
+ """
+ await session.execute(
+ update(MessagingEventReceipt)
+ .where(MessagingEventReceipt.id == receipt_id)
+ .values(handled_at=datetime.now(UTC))
+ )
+
+ async def prune(self, session: AsyncSession) -> int:
+ """Drop the bound tenant's receipts older than the retention window.
+
+ Opportunistic rather than scheduled, the way stale role leases are
+ cleared when the next agent takes a role. This backend has no janitor
+ to hang a sweep on, and inventing one for a single table would be a
+ larger change than the table deserves; running it off the traffic that
+ creates the rows keeps the work proportional to that traffic and needs
+ nothing started at boot.
+
+ Called after the event has been dispatched, not before it. A prune in
+ the claiming transaction would sit between a retry and the refusal it
+ is waiting for.
+ """
+ result = await session.execute(
+ delete(MessagingEventReceipt).where(
+ MessagingEventReceipt.received_at
+ < datetime.now(UTC) - RECEIPT_RETENTION
+ )
+ )
+ return int(result.rowcount or 0) # type: ignore[attr-defined]
diff --git a/core/switch_core/main.py b/core/switch_core/main.py
index 067e995d5..12ce05586 100644
--- a/core/switch_core/main.py
+++ b/core/switch_core/main.py
@@ -106,6 +106,7 @@ from switch_core.db.stores.external_user_store import ExternalUserStore
from switch_core.db.stores.invitation_store import InvitationStore
from switch_core.db.stores.media_store import MediaStore
from switch_core.db.stores.message_store import MessageStore
+from switch_core.db.stores.messaging_event_store import MessagingEventReceiptStore
from switch_core.db.stores.messaging_install_store import MessagingInstallStore
from switch_core.db.stores.package_store import PackageStore
from switch_core.db.stores.reference_store import ReferenceStore
@@ -506,6 +507,7 @@ async def run(config: SwitchConfig) -> None:
install_service = MessagingInstallService(
session_factory=session_factory,
store=MessagingInstallStore(),
+ receipts=MessagingEventReceiptStore(),
installers=installers,
lifecycle=collab_lifecycle,
public_origin=config.messaging_public_url,
diff --git a/core/switch_core/migrations/versions/b2e9d41c7f60_messaging_event_receipts.py b/core/switch_core/migrations/versions/b2e9d41c7f60_messaging_event_receipts.py
new file mode 100644
index 000000000..89249574e
--- /dev/null
+++ b/core/switch_core/migrations/versions/b2e9d41c7f60_messaging_event_receipts.py
@@ -0,0 +1,86 @@
+"""messaging_event_receipts
+
+A platform that retries is a platform that will eventually deliver the same
+event twice, and the visible failure is an agent answering one question twice
+in a customer's channel. This table is what makes the second delivery cheap to
+recognise: a row is written before the work starts, the unique index arbitrates
+between concurrent retries, and the loser stops.
+
+`received_at` carries an index of its own because pruning reads that column and
+nothing else. Receipts are only interesting for as long as the platform might
+still re-send, and without a sweep the table would grow with every message the
+busiest workspace ever sends.
+
+The `tenant_isolation` policy is written out below rather than inherited,
+because `265ed188ad6f` installed the policies as they stood then and a table
+added afterwards has to bring its own. The DDL is a verbatim copy of
+`switch_core/db/rls_ddl.py` as it stood when this migration was written,
+copied rather than imported for the reason every revision in this chain
+copies: a migration records a change that already happened, and importing the
+live module would let a later edit silently change what this one means.
+`tests/switch_core/db/test_frozen_ddl_matches_create_all.py` is what keeps the
+copy from drifting.
+
+Revision ID: b2e9d41c7f60
+Revises: a7f2c3e9b481
+Create Date: 2026-09-15 00:00:00.000000
+
+"""
+
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+from alembic import op
+
+revision: str = "b2e9d41c7f60"
+down_revision: str | None = "a7f2c3e9b481"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+TABLE = "messaging_event_receipts"
+POLICY_NAME = "tenant_isolation"
+
+ENABLE_RLS = f'ALTER TABLE "{TABLE}" ENABLE ROW LEVEL SECURITY'
+
+CREATE_POLICY = f"""CREATE POLICY {POLICY_NAME} ON "{TABLE}"
+ FOR ALL
+ USING ("tenant_id" = (SELECT require_tenant_id()))
+ WITH CHECK ("tenant_id" = (SELECT require_tenant_id()))"""
+
+DROP_POLICY = f'DROP POLICY IF EXISTS {POLICY_NAME} ON "{TABLE}"'
+
+
+def upgrade() -> None:
+ op.create_table(
+ TABLE,
+ sa.Column("id", sa.Text(), nullable=False),
+ sa.Column("tenant_id", sa.Text(), nullable=False),
+ sa.Column("platform", sa.Text(), nullable=False),
+ sa.Column("external_event_id", sa.Text(), nullable=False),
+ sa.Column(
+ "received_at",
+ sa.DateTime(timezone=True),
+ server_default=sa.text("now()"),
+ nullable=False,
+ ),
+ sa.Column("handled_at", sa.DateTime(timezone=True), nullable=True),
+ sa.PrimaryKeyConstraint("id"),
+ sa.ForeignKeyConstraint(
+ ["tenant_id"], ["tenants.id"], name="fk_messaging_event_receipts_tenant"
+ ),
+ sa.UniqueConstraint(
+ "tenant_id",
+ "platform",
+ "external_event_id",
+ name="uq_messaging_event_receipts_event",
+ ),
+ )
+ op.create_index("ix_messaging_event_receipts_received_at", TABLE, ["received_at"])
+ op.execute(ENABLE_RLS)
+ op.execute(CREATE_POLICY)
+
+
+def downgrade() -> None:
+ op.execute(DROP_POLICY)
+ op.drop_index("ix_messaging_event_receipts_received_at", table_name=TABLE)
+ op.drop_table(TABLE)
diff --git a/core/tests/switch_core/bridges/collaboration/test_install_service.py b/core/tests/switch_core/bridges/collaboration/test_install_service.py
index c7f0252f6..a6879881d 100644
--- a/core/tests/switch_core/bridges/collaboration/test_install_service.py
+++ b/core/tests/switch_core/bridges/collaboration/test_install_service.py
@@ -47,6 +47,7 @@ from switch_core.db.models import (
User,
)
from switch_core.db.session_scope import tenant_session
+from switch_core.db.stores.messaging_event_store import MessagingEventReceiptStore
from switch_core.db.stores.messaging_install_store import (
INSTALL_ACTIVE,
INSTALL_DISCONNECTED,
@@ -104,9 +105,15 @@ class _FakeInstaller(MessagingAppInstaller):
return None
def parse_webhook(
- self, *, endpoint: WebhookEndpoint, body: bytes
+ self, *, endpoint: WebhookEndpoint, headers: Mapping[str, str], body: bytes
) -> InboundWebhook:
- return InboundWebhook(envelope_type=endpoint, payload={}, handshake=None)
+ return InboundWebhook(
+ envelope_type=endpoint,
+ payload={},
+ handshake=None,
+ external_event_id=None,
+ delivery_attempt=0,
+ )
def workspace_of_event(self, payload: Mapping[str, object]) -> str:
return self.workspace_id
@@ -204,6 +211,7 @@ async def _fixture(harness: RLSHarness) -> _Fixture:
fixture.service = MessagingInstallService(
session_factory=harness.restricted,
store=MessagingInstallStore(),
+ receipts=MessagingEventReceiptStore(),
installers=installers,
lifecycle=fixture.lifecycle, # type: ignore[arg-type]
public_origin=_ORIGIN,
diff --git a/core/tests/switch_core/bridges/collaboration/test_install_webhook.py b/core/tests/switch_core/bridges/collaboration/test_install_webhook.py
index fd4244e92..7b72dc33f 100644
--- a/core/tests/switch_core/bridges/collaboration/test_install_webhook.py
+++ b/core/tests/switch_core/bridges/collaboration/test_install_webhook.py
@@ -19,18 +19,21 @@ socket: the bridge lifecycle and the adapter at the end of it.
from __future__ import annotations
+import asyncio
import hashlib
import hmac
import json
import time
import uuid
from dataclasses import dataclass
+from datetime import UTC, datetime, timedelta
from typing import Any
from urllib.parse import urlencode
import httpx
import pytest
from fastapi import FastAPI
+from sqlalchemy import select
from sqlalchemy.ext.asyncio import async_sessionmaker
from switch_core.bridges.collaboration.adapter import CollaborationAdapter
@@ -48,11 +51,16 @@ from switch_core.bridges.collaboration.slack.install import SlackAppInstaller
from switch_core.db.models import (
Client,
CollaborationBridge,
+ MessagingEventReceipt,
MessagingInstall,
Tenant,
User,
)
from switch_core.db.session_scope import tenant_session
+from switch_core.db.stores.messaging_event_store import (
+ RECEIPT_RETENTION,
+ MessagingEventReceiptStore,
+)
from switch_core.db.stores.messaging_install_store import MessagingInstallStore
from switch_core.tenant_context import current_tenant_id
from tests.conftest import RLSHarness
@@ -125,6 +133,29 @@ class _RecordingAdapter(_SocketOnlyAdapter):
self.tenants_bound.append(current_tenant_id())
+class _GatedAdapter(_SocketOnlyAdapter):
+ """An adapter that holds a dispatch open until it is let go.
+
+ What it buys is determinism. The race worth testing is a retry arriving
+ while the first delivery is still working — the whole reason the claim is
+ written before the dispatch rather than after — and an adapter that returns
+ instantly would let the first delivery finish before the second began,
+ proving only that a repeat is refused once the first is over.
+ """
+
+ def __init__(self) -> None:
+ self.dispatched: list[tuple[str, dict[str, Any]]] = []
+ self.entered = asyncio.Event()
+ self.release = asyncio.Event()
+
+ async def dispatch_event(
+ self, *, envelope_type: str, payload: dict[str, Any]
+ ) -> None:
+ self.dispatched.append((envelope_type, payload))
+ self.entered.set()
+ await self.release.wait()
+
+
class _FakeLifecycle:
def __init__(self, factory: async_sessionmaker) -> None:
self._factory = factory
@@ -247,6 +278,7 @@ async def _fixture(harness: RLSHarness) -> _Fixture:
fixture.service = MessagingInstallService(
session_factory=harness.restricted,
store=MessagingInstallStore(),
+ receipts=MessagingEventReceiptStore(),
installers=installers,
lifecycle=fixture.lifecycle, # type: ignore[arg-type]
public_origin=_ORIGIN,
@@ -271,6 +303,30 @@ def _event(workspace_id: str, text: str) -> bytes:
).encode()
+def _numbered_event(workspace_id: str, text: str, event_id: str) -> bytes:
+ """An event carrying the id Slack puts on everything it retries.
+
+ Separate from `_event` rather than an argument to it, so the tests above go
+ on exercising the unnumbered path — which is the one a slash command and an
+ interaction take, and which must dispatch rather than being refused for
+ having nothing to deduplicate by.
+ """
+ return json.dumps(
+ {
+ "type": "event_callback",
+ "team_id": workspace_id,
+ "event_id": event_id,
+ "event": {"type": "message", "text": text, "channel": "C1"},
+ }
+ ).encode()
+
+
+async def _receipts(factory: async_sessionmaker, tenant_id: str) -> list[Any]:
+ async with tenant_session(factory, tenant_id) as session:
+ rows = await session.execute(select(MessagingEventReceipt))
+ return list(rows.scalars())
+
+
def _uninstalled(workspace_id: str) -> bytes:
return json.dumps(
{
@@ -567,6 +623,197 @@ class TestABridgeThatCannotTakeEvents:
await fixture.service.deliver(target, event)
+class TestAnEventIsHandledOnce:
+ """The platform delivers at least once; the customer must hear once.
+
+ Slack allows three seconds to acknowledge and re-sends what it does not get
+ an answer to, so a deployment under load is told the same thing twice. The
+ payload of a retry is byte-identical to the original — the id is the only
+ thing that distinguishes it from someone saying the same words again — and
+ the visible failure is an agent answering one question twice in a
+ customer's channel.
+ """
+
+ async def test_a_retry_of_the_same_event_dispatches_once(
+ self, rls_harness: RLSHarness
+ ) -> None:
+ fixture = await _fixture(rls_harness)
+ body = _numbered_event(fixture.a.workspace_id, "hello", "Ev1")
+
+ first = await _post(fixture, events_path("slack"), body)
+ second = await _post(fixture, events_path("slack"), body)
+
+ assert (first.status_code, second.status_code) == (200, 200)
+ assert _texts(fixture.a.adapter) == ["hello"]
+
+ async def test_a_retry_arriving_mid_turn_loses_to_the_delivery_in_flight(
+ self, rls_harness: RLSHarness
+ ) -> None:
+ """The case the ordering exists for, and the only one that is a race.
+
+ Recording the receipt after the work instead of before would order the
+ two the wrong way round: both would find nothing claimed, both would
+ dispatch, and the duplicate would be noticed once it no longer
+ mattered.
+ """
+ fixture = await _fixture(rls_harness)
+ gated = _GatedAdapter()
+ fixture.lifecycle.adapters[fixture.a.bridge_id] = gated
+ body = _numbered_event(fixture.a.workspace_id, "hello", "Ev1")
+ event = fixture.service.authenticate(
+ platform="slack", endpoint="events", headers=_signed(body), body=body
+ )
+ target = await fixture.service.resolve(platform="slack", event=event)
+
+ in_flight = asyncio.create_task(fixture.service.deliver(target, event))
+ await gated.entered.wait()
+ await fixture.service.deliver(target, event)
+
+ assert len(gated.dispatched) == 1
+ gated.release.set()
+ await in_flight
+
+ async def test_an_event_with_no_id_is_dispatched_every_time(
+ self, rls_harness: RLSHarness
+ ) -> None:
+ """Not a weaker guarantee quietly accepted.
+
+ Slack numbers only the envelopes it retries, so one arriving without an
+ id arrives exactly once. Two of them are two real messages, and
+ refusing the second for having nothing to deduplicate by would drop a
+ customer's message on the floor.
+ """
+ fixture = await _fixture(rls_harness)
+ body = _event(fixture.a.workspace_id, "same words")
+
+ await _post(fixture, events_path("slack"), body)
+ await _post(fixture, events_path("slack"), body)
+
+ assert _texts(fixture.a.adapter) == ["same words", "same words"]
+ assert await _receipts(rls_harness.restricted, fixture.a.tenant_id) == []
+
+ async def test_one_tenants_event_ids_do_not_block_anothers(
+ self, rls_harness: RLSHarness
+ ) -> None:
+ """Uniqueness is per tenant, and this is why that is the shape to want.
+
+ Slack's ids are unique in its own namespace, so a deployment-wide index
+ would protect exactly as well — but it would let one customer's traffic
+ refuse another's, and a conflict would name a row the inserting tenant
+ cannot see.
+ """
+ fixture = await _fixture(rls_harness)
+
+ await _post(
+ fixture,
+ events_path("slack"),
+ _numbered_event(fixture.a.workspace_id, "a", "Ev-shared"),
+ )
+ await _post(
+ fixture,
+ events_path("slack"),
+ _numbered_event(fixture.b.workspace_id, "b", "Ev-shared"),
+ )
+
+ assert _texts(fixture.a.adapter) == ["a"]
+ assert _texts(fixture.b.adapter) == ["b"]
+
+ async def test_a_completed_delivery_is_recorded_as_handled(
+ self, rls_harness: RLSHarness
+ ) -> None:
+ """The absence of this is the useful half.
+
+ Claiming before the work chooses at-most-once: an event taken by a
+ process that then dies is not retried, because the platform has already
+ been told 200. `handled_at` is what keeps that loss findable — a
+ claimed receipt that never completed is a real event that reached
+ nobody.
+ """
+ fixture = await _fixture(rls_harness)
+
+ await _post(
+ fixture,
+ events_path("slack"),
+ _numbered_event(fixture.a.workspace_id, "hello", "Ev1"),
+ )
+
+ rows = await _receipts(rls_harness.restricted, fixture.a.tenant_id)
+ assert [(row.platform, row.external_event_id) for row in rows] == [
+ ("slack", "Ev1")
+ ]
+ assert rows[0].handled_at is not None
+
+ async def test_a_receipt_past_its_retention_is_pruned_by_the_next_event(
+ self, rls_harness: RLSHarness
+ ) -> None:
+ """Opportunistic, because this backend has no janitor to hang it on.
+
+ The table would otherwise grow with every message the busiest workspace
+ ever sends. Running the sweep off the traffic that creates the rows
+ keeps the work proportional to that traffic and needs nothing started
+ at boot.
+ """
+ fixture = await _fixture(rls_harness)
+ async with tenant_session(
+ rls_harness.restricted, fixture.a.tenant_id
+ ) as session:
+ session.add(
+ MessagingEventReceipt(
+ tenant_id=fixture.a.tenant_id,
+ platform="slack",
+ external_event_id="Ev-ancient",
+ received_at=datetime.now(UTC)
+ - RECEIPT_RETENTION
+ - timedelta(days=1),
+ )
+ )
+ await session.commit()
+
+ await _post(
+ fixture,
+ events_path("slack"),
+ _numbered_event(fixture.a.workspace_id, "hello", "Ev-fresh"),
+ )
+
+ rows = await _receipts(rls_harness.restricted, fixture.a.tenant_id)
+ assert [row.external_event_id for row in rows] == ["Ev-fresh"]
+
+ async def test_a_retry_of_an_event_nobody_holds_is_still_dropped_at_resolve(
+ self, rls_harness: RLSHarness
+ ) -> None:
+ """Deduplication sits after resolution, so an unknown workspace writes
+ no receipt at all — there is no tenant to write it as."""
+ fixture = await _fixture(rls_harness)
+
+ response = await _post(
+ fixture, events_path("slack"), _numbered_event("T-nobody", "x", "Ev1")
+ )
+
+ assert response.status_code == 200
+ assert await _receipts(rls_harness.restricted, fixture.a.tenant_id) == []
+
+ async def test_a_bridge_that_was_down_still_takes_the_retry(
+ self, rls_harness: RLSHarness
+ ) -> None:
+ """The 503 and the receipt compose, and this is the case that proves it.
+
+ A bridge mid-restart makes the event fail before any claim is written,
+ so the retry Slack sends in response finds nothing taken and is handled
+ normally. Claiming any earlier — before `resolve` — would turn a
+ transient outage into a permanently lost message.
+ """
+ fixture = await _fixture(rls_harness)
+ body = _numbered_event(fixture.a.workspace_id, "hello", "Ev1")
+ del fixture.lifecycle.adapters[fixture.a.bridge_id]
+
+ refused = await _post(fixture, events_path("slack"), body)
+ fixture.lifecycle.adapters[fixture.a.bridge_id] = fixture.a.adapter
+ retried = await _post(fixture, events_path("slack"), body)
+
+ assert (refused.status_code, retried.status_code) == (503, 200)
+ assert _texts(fixture.a.adapter) == ["hello"]
+
+
class TestThePlatformSayingTheInstallIsOver:
"""`app_uninstalled` arrives on the same URL as everything else."""
diff --git a/core/tests/switch_core/bridges/collaboration/test_slack_installer.py b/core/tests/switch_core/bridges/collaboration/test_slack_installer.py
index 5c7cb4c47..ea8695a44 100644
--- a/core/tests/switch_core/bridges/collaboration/test_slack_installer.py
+++ b/core/tests/switch_core/bridges/collaboration/test_slack_installer.py
@@ -21,6 +21,7 @@ from switch_core.bridges.collaboration.install import (
MessagingInstallerRegistry,
MessagingInstallError,
WebhookAuthenticityError,
+ WebhookEndpoint,
WebhookPayloadError,
events_path,
oauth_callback_path,
@@ -234,7 +235,7 @@ class TestParsingAWebhook:
}
).encode()
- parsed = installer.parse_webhook(endpoint="events", body=body)
+ parsed = installer.parse_webhook(endpoint="events", headers={}, body=body)
assert parsed.envelope_type == "events_api"
assert parsed.handshake is None
@@ -254,7 +255,7 @@ class TestParsingAWebhook:
"""
body = json.dumps({"type": "url_verification", "challenge": "abc123"}).encode()
- parsed = installer.parse_webhook(endpoint="events", body=body)
+ parsed = installer.parse_webhook(endpoint="events", headers={}, body=body)
assert parsed.handshake == "abc123"
@@ -263,7 +264,7 @@ class TestParsingAWebhook:
) -> None:
with pytest.raises(WebhookPayloadError, match="challenge"):
installer.parse_webhook(
- endpoint="events", body=b'{"type":"url_verification"}'
+ endpoint="events", headers={}, body=b'{"type":"url_verification"}'
)
def test_a_slash_command_is_its_form_fields(
@@ -273,7 +274,7 @@ class TestParsingAWebhook:
{"command": "/agents-status", "text": "", "team_id": "T1", "user_id": "U1"}
).encode()
- parsed = installer.parse_webhook(endpoint="commands", body=body)
+ parsed = installer.parse_webhook(endpoint="commands", headers={}, body=body)
assert parsed.envelope_type == "slash_commands"
assert parsed.payload["command"] == "/agents-status"
@@ -287,7 +288,7 @@ class TestParsingAWebhook:
inner = {"type": "block_actions", "team": {"id": "T1"}}
body = urlencode({"payload": json.dumps(inner)}).encode()
- parsed = installer.parse_webhook(endpoint="interactive", body=body)
+ parsed = installer.parse_webhook(endpoint="interactive", headers={}, body=body)
assert parsed.envelope_type == "interactive"
assert parsed.payload == inner
@@ -296,7 +297,106 @@ class TestParsingAWebhook:
self, installer: SlackAppInstaller
) -> None:
with pytest.raises(WebhookPayloadError, match="payload"):
- installer.parse_webhook(endpoint="interactive", body=b"other=1")
+ installer.parse_webhook(endpoint="interactive", headers={}, body=b"other=1")
+
+ def test_an_event_callback_carries_slacks_own_event_id(
+ self, installer: SlackAppInstaller
+ ) -> None:
+ """The id is the only thing two copies of one event have in common.
+
+ The payload of a retry is byte-identical to the original, so without
+ this nothing downstream could tell a re-send from somebody saying the
+ same words twice.
+ """
+ body = json.dumps(
+ {"type": "event_callback", "team_id": "T1", "event_id": "Ev123"}
+ ).encode()
+
+ parsed = installer.parse_webhook(endpoint="events", headers={}, body=body)
+
+ assert parsed.external_event_id == "Ev123"
+
+ @pytest.mark.parametrize(
+ "envelope",
+ [
+ pytest.param({"type": "event_callback", "team_id": "T1"}, id="absent"),
+ pytest.param(
+ {"type": "event_callback", "team_id": "T1", "event_id": ""},
+ id="empty",
+ ),
+ pytest.param(
+ {"type": "event_callback", "team_id": "T1", "event_id": 7},
+ id="not-a-string",
+ ),
+ ],
+ )
+ def test_an_event_callback_with_no_usable_id_deduplicates_by_nothing(
+ self, installer: SlackAppInstaller, envelope: dict[str, object]
+ ) -> None:
+ """No id means dispatch it, never refuse it.
+
+ An empty or missing `event_id` is not a reason to drop a real message
+ from a customer's channel — it only means there is nothing to key a
+ receipt on, so the event is handled the way an unnumbered one is.
+ """
+ parsed = installer.parse_webhook(
+ endpoint="events", headers={}, body=json.dumps(envelope).encode()
+ )
+
+ assert parsed.external_event_id is None
+
+ @pytest.mark.parametrize(
+ ("endpoint", "body"),
+ [
+ pytest.param(
+ "commands", urlencode({"team_id": "T1"}).encode(), id="command"
+ ),
+ pytest.param(
+ "interactive",
+ urlencode({"payload": json.dumps({"team": {"id": "T1"}})}).encode(),
+ id="interaction",
+ ),
+ ],
+ )
+ def test_the_form_endpoints_carry_no_id_because_slack_sends_them_once(
+ self, installer: SlackAppInstaller, endpoint: WebhookEndpoint, body: bytes
+ ) -> None:
+ """Slack retries neither, so there is nothing to deduplicate.
+
+ A command and an interaction are a person waiting on a dialog. Slack
+ sends each exactly once and puts no id on it.
+ """
+ parsed = installer.parse_webhook(endpoint=endpoint, headers={}, body=body)
+
+ assert parsed.external_event_id is None
+
+ @pytest.mark.parametrize(
+ ("headers", "expected"),
+ [
+ pytest.param({}, 0, id="first-delivery"),
+ pytest.param({"x-slack-retry-num": "2"}, 2, id="second-retry"),
+ pytest.param({"x-slack-retry-num": "nonsense"}, 0, id="unparseable"),
+ ],
+ )
+ def test_the_retry_count_is_read_as_a_hint_and_never_as_a_decision(
+ self,
+ installer: SlackAppInstaller,
+ headers: dict[str, str],
+ expected: int,
+ ) -> None:
+ """Unsigned input, so a bad value must not be able to refuse an event.
+
+ The retry header is outside the signature — which covers the body and
+ the timestamp — so a garbled or forged one degrades to zero rather than
+ raising. The worst a forgery achieves is a wrong number in a log line.
+ """
+ body = json.dumps(
+ {"type": "event_callback", "team_id": "T1", "event_id": "Ev1"}
+ ).encode()
+
+ parsed = installer.parse_webhook(endpoint="events", headers=headers, body=body)
+
+ assert parsed.delivery_attempt == expected
@pytest.mark.parametrize(
"body",
@@ -316,7 +416,7 @@ class TestParsingAWebhook:
rather than a shrug.
"""
with pytest.raises(WebhookPayloadError):
- installer.parse_webhook(endpoint="events", body=body)
+ installer.parse_webhook(endpoint="events", headers={}, body=body)
class TestWhichWorkspaceSentIt: