Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(messaging): add an idempotency key to the transactional email api (#120392)

Harley Alexanderf695a06f088b
diff --git a/products/messaging/backend/facade/contracts.py b/products/messaging/backend/facade/contracts.py
index f71a960b4d41..3a83dd790b75 100644
--- a/products/messaging/backend/facade/contracts.py
+++ b/products/messaging/backend/facade/contracts.py
@@ -66,6 +66,20 @@ class QuotaLimited(TransactionalEmailError):
code = "quota_limited"
+class InvalidIdempotencyKey(TransactionalEmailError):
+ code = "invalid_idempotency_key"
+
+
+class IdempotencyKeyInFlight(TransactionalEmailError):
+ """The first request with this key is still being queued. Safe to retry shortly."""
+
+ code = "idempotency_key_in_flight"
+
+
+class IdempotencyKeyReused(TransactionalEmailError):
+ code = "idempotency_key_reused"
+
+
class DeliveryUnavailable(Exception):
"""The email pipeline did not accept the send. Safe to retry."""
@@ -73,3 +87,5 @@ class DeliveryUnavailable(Exception):
@frozen
class QueuedEmail:
message_id: str
+ # True when an earlier request with the same idempotency key already queued this email.
+ replayed: bool = False
diff --git a/products/messaging/backend/presentation/views/message_templates.py b/products/messaging/backend/presentation/views/message_templates.py
index 52abf0918117..c2e307bb8b6c 100644
--- a/products/messaging/backend/presentation/views/message_templates.py
+++ b/products/messaging/backend/presentation/views/message_templates.py
@@ -280,6 +280,16 @@ TEMPLATE_ID_PARAMETER = OpenApiParameter(
description="A UUID string identifying this message template.",
)
+IDEMPOTENCY_KEY_PARAMETER = OpenApiParameter(
+ name="Idempotency-Key",
+ type=OpenApiTypes.STR,
+ location=OpenApiParameter.HEADER,
+ required=False,
+ description="A unique key for this email, 1 to 255 visible ASCII characters. Retrying with the same key and "
+ "the same request within 24 hours sends nothing new and returns the first send's `message_id`. Reusing a key "
+ "for a different request returns a 422.",
+)
+
TRANSACTIONAL_EMAIL_API_FLAG = "workflows-transactional-email-api"
@@ -323,10 +333,14 @@ class SendTransactionalEmailResponseSerializer(serializers.Serializer):
message_id = serializers.CharField(
help_text="ID of the queued send. Delivery and engagement are reported against the template asynchronously."
)
+ replayed = serializers.BooleanField(
+ help_text="True when an earlier request with the same `Idempotency-Key` already queued this email. "
+ "No new email was sent, and `message_id` is the earlier send's ID."
+ )
class TransactionalEmailRejected(exceptions.APIException):
- STATUS_BY_CODE = {"quota_limited": 429}
+ STATUS_BY_CODE = {"quota_limited": 429, "idempotency_key_in_flight": 409, "idempotency_key_reused": 422}
# The caller has to change the request. Without this the error handler reports a server error.
default_type = "invalid_request"
@@ -606,9 +620,12 @@ class MessageTemplatesViewSet(
responses={
202: OpenApiResponse(response=SendTransactionalEmailResponseSerializer, description="Email queued"),
400: OpenApiResponse(
- description="The template isn't transactional or its email is paused, or the recipient can't be used"
+ description="The template isn't transactional or its email is paused, the recipient can't be used, "
+ "or the Idempotency-Key is invalid"
),
404: OpenApiResponse(description="Template not found"),
+ 409: OpenApiResponse(description="A request with the same Idempotency-Key is still in progress"),
+ 422: OpenApiResponse(description="The Idempotency-Key was already used for a different request"),
429: OpenApiResponse(description="Rate limited, or the project is over its email quota"),
503: OpenApiResponse(description="The email pipeline did not confirm the send"),
},
@@ -616,9 +633,10 @@ class MessageTemplatesViewSet(
description=(
"Send one transactional email from this template to one recipient. The template must be made "
"transactional first. `message_data` fills the template's `{{ variables.* }}` placeholders. "
- "Transactional emails skip opt-outs and frequency caps."
+ "Transactional emails skip opt-outs and frequency caps. Send an `Idempotency-Key` header to retry "
+ "safely without sending the email twice."
),
- parameters=[TEMPLATE_ID_PARAMETER],
+ parameters=[TEMPLATE_ID_PARAMETER, IDEMPOTENCY_KEY_PARAMETER],
)
@action(detail=True, methods=["POST"], pagination_class=None)
def send(self, request: Any, *args: Any, **kwargs: Any) -> Response:
@@ -628,7 +646,8 @@ class MessageTemplatesViewSet(
if template.transactional_hog_flow_id:
self._require_workflow_editor(template.transactional_hog_flow_id)
data = request.validated_data
- outcome: dict[str, Any] = {"outcome": "error"}
+ idempotency_key = request.headers.get("Idempotency-Key")
+ outcome: dict[str, Any] = {"outcome": "error", "has_idempotency_key": idempotency_key is not None}
try:
queued = send_transactional_email(
self.team,
@@ -636,13 +655,14 @@ class MessageTemplatesViewSet(
to=data["to"],
distinct_id=(data.get("identifiers") or {}).get("distinct_id") or None,
message_data=data["message_data"],
+ idempotency_key=idempotency_key,
)
- outcome = {"outcome": "queued"}
+ outcome["outcome"] = "replayed" if queued.replayed else "queued"
except TransactionalEmailError as e:
- outcome = {"outcome": "rejected", "error_code": e.code}
+ outcome.update({"outcome": "rejected", "error_code": e.code})
raise TransactionalEmailRejected(e)
except DeliveryUnavailable:
- outcome = {"outcome": "unavailable"}
+ outcome["outcome"] = "unavailable"
raise TransactionalEmailUnavailable() from None
finally:
self._report_transactional_email_send(request, template, outcome)
diff --git a/products/messaging/backend/services/transactional_email.py b/products/messaging/backend/services/transactional_email.py
index 28c162a366a5..5463d675710a 100644
--- a/products/messaging/backend/services/transactional_email.py
+++ b/products/messaging/backend/services/transactional_email.py
@@ -1,6 +1,7 @@
"""Transactional emails: a template made sendable over the API. Each one is backed by a workflow that runs once
per send, so sends get the workflow's metrics, run history and versions."""
+import re
from typing import Any
from uuid import UUID
@@ -16,6 +17,9 @@ from posthog.plugins.plugin_server_api import (
from products.messaging.backend.facade.contracts import (
DeliveryUnavailable,
+ IdempotencyKeyInFlight,
+ IdempotencyKeyReused,
+ InvalidIdempotencyKey,
InvalidRecipient,
MarketingTemplate,
QueuedEmail,
@@ -38,6 +42,8 @@ from products.workflows.backend.facade.transactional_emails import (
logger = structlog.get_logger(__name__)
+IDEMPOTENCY_KEY_PATTERN = re.compile(r"[!-~]{1,255}")
+
def check_sendable(team_id: int, *, template_type: str, content: dict, message_category_id: UUID | None) -> None:
"""Raise when a template with these fields can't be a transactional email."""
@@ -195,8 +201,18 @@ def _transactional_workflow_id(team_id: int, template_id: UUID) -> UUID:
def send_transactional_email(
- team: Team, template_id: UUID, *, to: str, distinct_id: str | None, message_data: dict[str, Any]
+ team: Team,
+ template_id: UUID,
+ *,
+ to: str,
+ distinct_id: str | None,
+ message_data: dict[str, Any],
+ idempotency_key: str | None = None,
) -> QueuedEmail:
+ if idempotency_key is not None and not IDEMPOTENCY_KEY_PATTERN.fullmatch(idempotency_key):
+ raise InvalidIdempotencyKey(
+ "The Idempotency-Key header must be 1 to 255 visible ASCII characters, with no spaces."
+ )
# Defense in depth: the workflow reads the recipient from an event property, which Liquid outputs as typed.
if "{" in to or "}" in to:
raise InvalidRecipient("The recipient address can't contain { or }.")
@@ -208,6 +224,9 @@ def send_transactional_email(
"distinct_id": distinct_id or to,
"variables": message_data,
}
+ if idempotency_key is not None:
+ # The CDP service holds the key, because only it knows whether the email was queued.
+ payload["idempotency_key"] = idempotency_key
try:
response = run_transactional_workflow(team.id, str(hog_flow_id), payload)
except Exception as e:
@@ -217,6 +236,14 @@ def send_transactional_email(
if response.status_code != 202:
if response.status_code == 429:
raise QuotaLimited("The project is over its email quota.")
+ if response.status_code == 409:
+ raise IdempotencyKeyInFlight(
+ "A request with this Idempotency-Key is still being processed. Retry in a few seconds."
+ )
+ if response.status_code == 422:
+ raise IdempotencyKeyReused(
+ "This Idempotency-Key was already used for a different request. Use a new key for a new email."
+ )
logger.error(
"transactional_email_queue_rejected",
team_id=team.id,
@@ -225,4 +252,5 @@ def send_transactional_email(
)
raise DeliveryUnavailable()
- return QueuedEmail(message_id=str(response.json()["invocation_id"]))
+ body = response.json()
+ return QueuedEmail(message_id=str(body["invocation_id"]), replayed=bool(body.get("replayed", False)))
diff --git a/products/messaging/backend/tests/api/test_transactional_email.py b/products/messaging/backend/tests/api/test_transactional_email.py
index 69f3f78ecb77..a6fdf900ecf9 100644
--- a/products/messaging/backend/tests/api/test_transactional_email.py
+++ b/products/messaging/backend/tests/api/test_transactional_email.py
@@ -97,7 +97,7 @@ class TestTransactionalEmail(APIBaseTest):
response = self._psak_client(["hog_flow:write"]).post(self.url, body, format="json")
assert response.status_code == status.HTTP_202_ACCEPTED, response.json()
- assert response.json() == {"message_id": "invocation-1"}
+ assert response.json() == {"message_id": "invocation-1", "replayed": False}
assert mock_post.call_args.args[0].endswith(
f"/api/projects/{self.team.id}/hog_flows/{self.hog_flow_id}/transactional_email"
)
@@ -107,6 +107,19 @@ class TestTransactionalEmail(APIBaseTest):
"variables": {"link": "https://example.com/reset", "nickname": None},
}
+ def test_passes_the_idempotency_key_to_the_pipeline_and_reports_a_replay(self, mock_post, _flag, capture):
+ mock_post.return_value = MagicMock(
+ status_code=202, json=lambda: {"invocation_id": "first-send", "replayed": True}
+ )
+
+ response = self.client.post(self.url, self.body, format="json", headers={"Idempotency-Key": "reset:user-1:42"})
+
+ assert response.status_code == status.HTTP_202_ACCEPTED, response.json()
+ assert response.json() == {"message_id": "first-send", "replayed": True}
+ assert mock_post.call_args.kwargs["json"]["idempotency_key"] == "reset:user-1:42"
+ assert capture.call_args.kwargs["properties"]["outcome"] == "replayed"
+ assert capture.call_args.kwargs["properties"]["has_idempotency_key"] is True
+
@parameterized.expand(
[
("send_with_write_scope", "post", "send/", ["hog_flow:write"], status.HTTP_202_ACCEPTED),
@@ -139,10 +152,12 @@ class TestTransactionalEmail(APIBaseTest):
("workflow_archived", "transactional_email_paused"),
("category_turned_marketing", "marketing_template"),
("liquid_in_recipient", "invalid_recipient"),
+ ("idempotency_key_with_spaces", "invalid_idempotency_key"),
+ ("idempotency_key_too_long", "invalid_idempotency_key"),
]
)
def test_rejects_a_send_it_cannot_make(self, mock_post, _flag, _capture, case, expected_code):
- url, body = self.url, dict(self.body)
+ url, body, headers = self.url, dict(self.body), {}
if case == "not_transactional":
url = f"/api/projects/{self.team.id}/messaging_templates/{self._template().id}/send/"
elif case == "workflow_archived":
@@ -158,8 +173,12 @@ class TestTransactionalEmail(APIBaseTest):
MessageCategory.objects.filter(id=category.id).update(category_type=MessageCategoryType.MARKETING)
elif case == "liquid_in_recipient":
body["to"] = "{{person.properties.email}}@example.com"
+ elif case == "idempotency_key_with_spaces":
+ headers["Idempotency-Key"] = "reset user 1"
+ elif case == "idempotency_key_too_long":
+ headers["Idempotency-Key"] = "k" * 256
- response = self.client.post(url, body, format="json")
+ response = self.client.post(url, body, format="json", headers=headers)
assert response.status_code == status.HTTP_400_BAD_REQUEST, response.json()
assert response.json()["code"] == expected_code
@@ -234,13 +253,20 @@ class TestTransactionalEmail(APIBaseTest):
assert edited.json()["transactional_hog_flow_id"] is None
assert remade.json()["transactional_hog_flow_id"] not in (None, str(self.hog_flow_id))
- def test_reports_quota_limited_teams(self, mock_post, _flag, _capture):
- mock_post.return_value = MagicMock(status_code=429, json=lambda: {"error": "quota_limited"})
+ @parameterized.expand(
+ [
+ ("quota_limited", 429),
+ ("idempotency_key_in_flight", 409),
+ ("idempotency_key_reused", 422),
+ ]
+ )
+ def test_reports_a_send_the_pipeline_refuses(self, mock_post, _flag, _capture, code, status_code):
+ mock_post.return_value = MagicMock(status_code=status_code, json=lambda: {"error": code})
- response = self.client.post(self.url, self.body, format="json")
+ response = self.client.post(self.url, self.body, format="json", headers={"Idempotency-Key": "key-1"})
- assert response.status_code == status.HTTP_429_TOO_MANY_REQUESTS
- assert response.json()["code"] == "quota_limited"
+ assert response.status_code == status_code
+ assert response.json()["code"] == code
@parameterized.expand([("send", "send/"), ("make_transactional", "make_transactional/")])
def test_is_hidden_while_the_flag_is_off(self, mock_post, flag, _capture, _name, suffix):
diff --git a/products/messaging/frontend/generated/api.schemas.ts b/products/messaging/frontend/generated/api.schemas.ts
index 1d6365fe58a4..ee1430abd4c7 100644
--- a/products/messaging/frontend/generated/api.schemas.ts
+++ b/products/messaging/frontend/generated/api.schemas.ts
@@ -546,6 +546,8 @@ export interface SendTransactionalEmailApi {
export interface SendTransactionalEmailResponseApi {
/** ID of the queued send. Delivery and engagement are reported against the template asynchronously. */
message_id: string
+ /** True when an earlier request with the same `Idempotency-Key` already queued this email. No new email was sent, and `message_id` is the earlier send's ID. */
+ replayed: boolean
}
export type MessagingCategoriesListParams = {
diff --git a/products/messaging/frontend/generated/api.ts b/products/messaging/frontend/generated/api.ts
index 6f4a2bd8cc79..17f050bec0ab 100644
--- a/products/messaging/frontend/generated/api.ts
+++ b/products/messaging/frontend/generated/api.ts
@@ -752,7 +752,7 @@ export const getMessagingTemplatesSendCreateUrl = (projectId: string, id: string
}
/**
- * Send one transactional email from this template to one recipient. The template must be made transactional first. `message_data` fills the template's `{{ variables.* }}` placeholders. Transactional emails skip opt-outs and frequency caps.
+ * Send one transactional email from this template to one recipient. The template must be made transactional first. `message_data` fills the template's `{{ variables.* }}` placeholders. Transactional emails skip opt-outs and frequency caps. Send an `Idempotency-Key` header to retry safely without sending the email twice.
* @summary Send a transactional email
*/
export const messagingTemplatesSendCreate = async (
diff --git a/products/messaging/frontend/generated/api.zod.ts b/products/messaging/frontend/generated/api.zod.ts
index 748b379518df..4b463f034491 100644
--- a/products/messaging/frontend/generated/api.zod.ts
+++ b/products/messaging/frontend/generated/api.zod.ts
@@ -868,7 +868,7 @@ export const MessagingTemplatesMakeTransactionalCreateBody = /* @__PURE__ */ zod
})
/**
- * Send one transactional email from this template to one recipient. The template must be made transactional first. `message_data` fills the template's `{{ variables.* }}` placeholders. Transactional emails skip opt-outs and frequency caps.
+ * Send one transactional email from this template to one recipient. The template must be made transactional first. `message_data` fills the template's `{{ variables.* }}` placeholders. Transactional emails skip opt-outs and frequency caps. Send an `Idempotency-Key` header to retry safely without sending the email twice.
* @summary Send a transactional email
*/
export const messagingTemplatesSendCreateBodyIdentifiersOneDistinctIdMax = 400
diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts
index 24de5d4b8b9d..8c39ab1028c3 100644
--- a/services/mcp/src/api/generated.ts
+++ b/services/mcp/src/api/generated.ts
@@ -103242,6 +103242,8 @@ export namespace Schemas {
export interface SendTransactionalEmailResponse {
/** ID of the queued send. Delivery and engagement are reported against the template asynchronously. */
message_id: string;
+ /** True when an earlier request with the same `Idempotency-Key` already queued this email. No new email was sent, and `message_id` is the earlier send's ID. */
+ replayed: boolean;
}
export interface ServiceAccountAccessUpdate {