:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(messaging): add an idempotency key to the transactional email api (#120392)
Harley Alexanderf695a06f088b
diff --git a/products/messaging/backend/facade/contracts.py b/products/messaging/backend/facade/contracts.pyindex f71a960b4d41..3a83dd790b75 100644--- a/products/messaging/backend/facade/contracts.py+++ b/products/messaging/backend/facade/contracts.py@@ -66,6 +66,20 @@ class QuotaLimited(TransactionalEmailError):code = "quota_limited"+class InvalidIdempotencyKey(TransactionalEmailError):+ code = "invalid_idempotency_key"+++class IdempotencyKeyInFlight(TransactionalEmailError):+ """The first request with this key is still being queued. Safe to retry shortly."""++ code = "idempotency_key_in_flight"+++class IdempotencyKeyReused(TransactionalEmailError):+ code = "idempotency_key_reused"++class DeliveryUnavailable(Exception):"""The email pipeline did not accept the send. Safe to retry."""@@ -73,3 +87,5 @@ class DeliveryUnavailable(Exception):@frozenclass QueuedEmail:message_id: str+ # True when an earlier request with the same idempotency key already queued this email.+ replayed: bool = Falsediff --git a/products/messaging/backend/presentation/views/message_templates.py b/products/messaging/backend/presentation/views/message_templates.pyindex 52abf0918117..c2e307bb8b6c 100644--- a/products/messaging/backend/presentation/views/message_templates.py+++ b/products/messaging/backend/presentation/views/message_templates.py@@ -280,6 +280,16 @@ TEMPLATE_ID_PARAMETER = OpenApiParameter(description="A UUID string identifying this message template.",)+IDEMPOTENCY_KEY_PARAMETER = OpenApiParameter(+ name="Idempotency-Key",+ type=OpenApiTypes.STR,+ location=OpenApiParameter.HEADER,+ required=False,+ description="A unique key for this email, 1 to 255 visible ASCII characters. Retrying with the same key and "+ "the same request within 24 hours sends nothing new and returns the first send's `message_id`. Reusing a key "+ "for a different request returns a 422.",+)+TRANSACTIONAL_EMAIL_API_FLAG = "workflows-transactional-email-api"@@ -323,10 +333,14 @@ class SendTransactionalEmailResponseSerializer(serializers.Serializer):message_id = serializers.CharField(help_text="ID of the queued send. Delivery and engagement are reported against the template asynchronously.")+ replayed = serializers.BooleanField(+ help_text="True when an earlier request with the same `Idempotency-Key` already queued this email. "+ "No new email was sent, and `message_id` is the earlier send's ID."+ )class TransactionalEmailRejected(exceptions.APIException):- STATUS_BY_CODE = {"quota_limited": 429}+ STATUS_BY_CODE = {"quota_limited": 429, "idempotency_key_in_flight": 409, "idempotency_key_reused": 422}# The caller has to change the request. Without this the error handler reports a server error.default_type = "invalid_request"@@ -606,9 +620,12 @@ class MessageTemplatesViewSet(responses={202: OpenApiResponse(response=SendTransactionalEmailResponseSerializer, description="Email queued"),400: OpenApiResponse(- description="The template isn't transactional or its email is paused, or the recipient can't be used"+ description="The template isn't transactional or its email is paused, the recipient can't be used, "+ "or the Idempotency-Key is invalid"),404: OpenApiResponse(description="Template not found"),+ 409: OpenApiResponse(description="A request with the same Idempotency-Key is still in progress"),+ 422: OpenApiResponse(description="The Idempotency-Key was already used for a different request"),429: OpenApiResponse(description="Rate limited, or the project is over its email quota"),503: OpenApiResponse(description="The email pipeline did not confirm the send"),},@@ -616,9 +633,10 @@ class MessageTemplatesViewSet(description=("Send one transactional email from this template to one recipient. The template must be made ""transactional first. `message_data` fills the template's `{{ variables.* }}` placeholders. "- "Transactional emails skip opt-outs and frequency caps."+ "Transactional emails skip opt-outs and frequency caps. Send an `Idempotency-Key` header to retry "+ "safely without sending the email twice."),- parameters=[TEMPLATE_ID_PARAMETER],+ parameters=[TEMPLATE_ID_PARAMETER, IDEMPOTENCY_KEY_PARAMETER],)@action(detail=True, methods=["POST"], pagination_class=None)def send(self, request: Any, *args: Any, **kwargs: Any) -> Response:@@ -628,7 +646,8 @@ class MessageTemplatesViewSet(if template.transactional_hog_flow_id:self._require_workflow_editor(template.transactional_hog_flow_id)data = request.validated_data- outcome: dict[str, Any] = {"outcome": "error"}+ idempotency_key = request.headers.get("Idempotency-Key")+ outcome: dict[str, Any] = {"outcome": "error", "has_idempotency_key": idempotency_key is not None}try:queued = send_transactional_email(self.team,@@ -636,13 +655,14 @@ class MessageTemplatesViewSet(to=data["to"],distinct_id=(data.get("identifiers") or {}).get("distinct_id") or None,message_data=data["message_data"],+ idempotency_key=idempotency_key,)- outcome = {"outcome": "queued"}+ outcome["outcome"] = "replayed" if queued.replayed else "queued"except TransactionalEmailError as e:- outcome = {"outcome": "rejected", "error_code": e.code}+ outcome.update({"outcome": "rejected", "error_code": e.code})raise TransactionalEmailRejected(e)except DeliveryUnavailable:- outcome = {"outcome": "unavailable"}+ outcome["outcome"] = "unavailable"raise TransactionalEmailUnavailable() from Nonefinally:self._report_transactional_email_send(request, template, outcome)diff --git a/products/messaging/backend/services/transactional_email.py b/products/messaging/backend/services/transactional_email.pyindex 28c162a366a5..5463d675710a 100644--- a/products/messaging/backend/services/transactional_email.py+++ b/products/messaging/backend/services/transactional_email.py@@ -1,6 +1,7 @@"""Transactional emails: a template made sendable over the API. Each one is backed by a workflow that runs onceper send, so sends get the workflow's metrics, run history and versions."""+import refrom typing import Anyfrom uuid import UUID@@ -16,6 +17,9 @@ from posthog.plugins.plugin_server_api import (from products.messaging.backend.facade.contracts import (DeliveryUnavailable,+ IdempotencyKeyInFlight,+ IdempotencyKeyReused,+ InvalidIdempotencyKey,InvalidRecipient,MarketingTemplate,QueuedEmail,@@ -38,6 +42,8 @@ from products.workflows.backend.facade.transactional_emails import (logger = structlog.get_logger(__name__)+IDEMPOTENCY_KEY_PATTERN = re.compile(r"[!-~]{1,255}")+def check_sendable(team_id: int, *, template_type: str, content: dict, message_category_id: UUID | None) -> None:"""Raise when a template with these fields can't be a transactional email."""@@ -195,8 +201,18 @@ def _transactional_workflow_id(team_id: int, template_id: UUID) -> UUID:def send_transactional_email(- team: Team, template_id: UUID, *, to: str, distinct_id: str | None, message_data: dict[str, Any]+ team: Team,+ template_id: UUID,+ *,+ to: str,+ distinct_id: str | None,+ message_data: dict[str, Any],+ idempotency_key: str | None = None,) -> QueuedEmail:+ if idempotency_key is not None and not IDEMPOTENCY_KEY_PATTERN.fullmatch(idempotency_key):+ raise InvalidIdempotencyKey(+ "The Idempotency-Key header must be 1 to 255 visible ASCII characters, with no spaces."+ )# Defense in depth: the workflow reads the recipient from an event property, which Liquid outputs as typed.if "{" in to or "}" in to:raise InvalidRecipient("The recipient address can't contain { or }.")@@ -208,6 +224,9 @@ def send_transactional_email("distinct_id": distinct_id or to,"variables": message_data,}+ if idempotency_key is not None:+ # The CDP service holds the key, because only it knows whether the email was queued.+ payload["idempotency_key"] = idempotency_keytry:response = run_transactional_workflow(team.id, str(hog_flow_id), payload)except Exception as e:@@ -217,6 +236,14 @@ def send_transactional_email(if response.status_code != 202:if response.status_code == 429:raise QuotaLimited("The project is over its email quota.")+ if response.status_code == 409:+ raise IdempotencyKeyInFlight(+ "A request with this Idempotency-Key is still being processed. Retry in a few seconds."+ )+ if response.status_code == 422:+ raise IdempotencyKeyReused(+ "This Idempotency-Key was already used for a different request. Use a new key for a new email."+ )logger.error("transactional_email_queue_rejected",team_id=team.id,@@ -225,4 +252,5 @@ def send_transactional_email()raise DeliveryUnavailable()- return QueuedEmail(message_id=str(response.json()["invocation_id"]))+ body = response.json()+ return QueuedEmail(message_id=str(body["invocation_id"]), replayed=bool(body.get("replayed", False)))diff --git a/products/messaging/backend/tests/api/test_transactional_email.py b/products/messaging/backend/tests/api/test_transactional_email.pyindex 69f3f78ecb77..a6fdf900ecf9 100644--- a/products/messaging/backend/tests/api/test_transactional_email.py+++ b/products/messaging/backend/tests/api/test_transactional_email.py@@ -97,7 +97,7 @@ class TestTransactionalEmail(APIBaseTest):response = self._psak_client(["hog_flow:write"]).post(self.url, body, format="json")assert response.status_code == status.HTTP_202_ACCEPTED, response.json()- assert response.json() == {"message_id": "invocation-1"}+ assert response.json() == {"message_id": "invocation-1", "replayed": False}assert mock_post.call_args.args[0].endswith(f"/api/projects/{self.team.id}/hog_flows/{self.hog_flow_id}/transactional_email")@@ -107,6 +107,19 @@ class TestTransactionalEmail(APIBaseTest):"variables": {"link": "https://example.com/reset", "nickname": None},}+ def test_passes_the_idempotency_key_to_the_pipeline_and_reports_a_replay(self, mock_post, _flag, capture):+ mock_post.return_value = MagicMock(+ status_code=202, json=lambda: {"invocation_id": "first-send", "replayed": True}+ )++ response = self.client.post(self.url, self.body, format="json", headers={"Idempotency-Key": "reset:user-1:42"})++ assert response.status_code == status.HTTP_202_ACCEPTED, response.json()+ assert response.json() == {"message_id": "first-send", "replayed": True}+ assert mock_post.call_args.kwargs["json"]["idempotency_key"] == "reset:user-1:42"+ assert capture.call_args.kwargs["properties"]["outcome"] == "replayed"+ assert capture.call_args.kwargs["properties"]["has_idempotency_key"] is True+@parameterized.expand([("send_with_write_scope", "post", "send/", ["hog_flow:write"], status.HTTP_202_ACCEPTED),@@ -139,10 +152,12 @@ class TestTransactionalEmail(APIBaseTest):("workflow_archived", "transactional_email_paused"),("category_turned_marketing", "marketing_template"),("liquid_in_recipient", "invalid_recipient"),+ ("idempotency_key_with_spaces", "invalid_idempotency_key"),+ ("idempotency_key_too_long", "invalid_idempotency_key"),])def test_rejects_a_send_it_cannot_make(self, mock_post, _flag, _capture, case, expected_code):- url, body = self.url, dict(self.body)+ url, body, headers = self.url, dict(self.body), {}if case == "not_transactional":url = f"/api/projects/{self.team.id}/messaging_templates/{self._template().id}/send/"elif case == "workflow_archived":@@ -158,8 +173,12 @@ class TestTransactionalEmail(APIBaseTest):MessageCategory.objects.filter(id=category.id).update(category_type=MessageCategoryType.MARKETING)elif case == "liquid_in_recipient":body["to"] = "{{person.properties.email}}@example.com"+ elif case == "idempotency_key_with_spaces":+ headers["Idempotency-Key"] = "reset user 1"+ elif case == "idempotency_key_too_long":+ headers["Idempotency-Key"] = "k" * 256- response = self.client.post(url, body, format="json")+ response = self.client.post(url, body, format="json", headers=headers)assert response.status_code == status.HTTP_400_BAD_REQUEST, response.json()assert response.json()["code"] == expected_code@@ -234,13 +253,20 @@ class TestTransactionalEmail(APIBaseTest):assert edited.json()["transactional_hog_flow_id"] is Noneassert remade.json()["transactional_hog_flow_id"] not in (None, str(self.hog_flow_id))- def test_reports_quota_limited_teams(self, mock_post, _flag, _capture):- mock_post.return_value = MagicMock(status_code=429, json=lambda: {"error": "quota_limited"})+ @parameterized.expand(+ [+ ("quota_limited", 429),+ ("idempotency_key_in_flight", 409),+ ("idempotency_key_reused", 422),+ ]+ )+ def test_reports_a_send_the_pipeline_refuses(self, mock_post, _flag, _capture, code, status_code):+ mock_post.return_value = MagicMock(status_code=status_code, json=lambda: {"error": code})- response = self.client.post(self.url, self.body, format="json")+ response = self.client.post(self.url, self.body, format="json", headers={"Idempotency-Key": "key-1"})- assert response.status_code == status.HTTP_429_TOO_MANY_REQUESTS- assert response.json()["code"] == "quota_limited"+ assert response.status_code == status_code+ assert response.json()["code"] == code@parameterized.expand([("send", "send/"), ("make_transactional", "make_transactional/")])def test_is_hidden_while_the_flag_is_off(self, mock_post, flag, _capture, _name, suffix):diff --git a/products/messaging/frontend/generated/api.schemas.ts b/products/messaging/frontend/generated/api.schemas.tsindex 1d6365fe58a4..ee1430abd4c7 100644--- a/products/messaging/frontend/generated/api.schemas.ts+++ b/products/messaging/frontend/generated/api.schemas.ts@@ -546,6 +546,8 @@ export interface SendTransactionalEmailApi {export interface SendTransactionalEmailResponseApi {/** ID of the queued send. Delivery and engagement are reported against the template asynchronously. */message_id: string+ /** True when an earlier request with the same `Idempotency-Key` already queued this email. No new email was sent, and `message_id` is the earlier send's ID. */+ replayed: boolean}export type MessagingCategoriesListParams = {diff --git a/products/messaging/frontend/generated/api.ts b/products/messaging/frontend/generated/api.tsindex 6f4a2bd8cc79..17f050bec0ab 100644--- a/products/messaging/frontend/generated/api.ts+++ b/products/messaging/frontend/generated/api.ts@@ -752,7 +752,7 @@ export const getMessagingTemplatesSendCreateUrl = (projectId: string, id: string}/**- * Send one transactional email from this template to one recipient. The template must be made transactional first. `message_data` fills the template's `{{ variables.* }}` placeholders. Transactional emails skip opt-outs and frequency caps.+ * Send one transactional email from this template to one recipient. The template must be made transactional first. `message_data` fills the template's `{{ variables.* }}` placeholders. Transactional emails skip opt-outs and frequency caps. Send an `Idempotency-Key` header to retry safely without sending the email twice.* @summary Send a transactional email*/export const messagingTemplatesSendCreate = async (diff --git a/products/messaging/frontend/generated/api.zod.ts b/products/messaging/frontend/generated/api.zod.tsindex 748b379518df..4b463f034491 100644--- a/products/messaging/frontend/generated/api.zod.ts+++ b/products/messaging/frontend/generated/api.zod.ts@@ -868,7 +868,7 @@ export const MessagingTemplatesMakeTransactionalCreateBody = /* @__PURE__ */ zod})/**- * Send one transactional email from this template to one recipient. The template must be made transactional first. `message_data` fills the template's `{{ variables.* }}` placeholders. Transactional emails skip opt-outs and frequency caps.+ * Send one transactional email from this template to one recipient. The template must be made transactional first. `message_data` fills the template's `{{ variables.* }}` placeholders. Transactional emails skip opt-outs and frequency caps. Send an `Idempotency-Key` header to retry safely without sending the email twice.* @summary Send a transactional email*/export const messagingTemplatesSendCreateBodyIdentifiersOneDistinctIdMax = 400diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.tsindex 24de5d4b8b9d..8c39ab1028c3 100644--- a/services/mcp/src/api/generated.ts+++ b/services/mcp/src/api/generated.ts@@ -103242,6 +103242,8 @@ export namespace Schemas {export interface SendTransactionalEmailResponse {/** ID of the queued send. Delivery and engagement are reported against the template asynchronously. */message_id: string;+ /** True when an earlier request with the same `Idempotency-Key` already queued this email. No new email was sent, and `message_id` is the earlier send's ID. */+ replayed: boolean;}export interface ServiceAccountAccessUpdate {