:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(messaging): refuse transactional sends the email worker would skip (#123019)
Harley Alexander02ce58f85782
diff --git a/nodejs/src/cdp/cdp-api.serial.test.ts b/nodejs/src/cdp/cdp-api.serial.test.tsindex 6a5914f4fcc1..1fcc77b91803 100644--- a/nodejs/src/cdp/cdp-api.serial.test.ts+++ b/nodejs/src/cdp/cdp-api.serial.test.ts@@ -2146,6 +2146,10 @@ describe('CDP API', () => {hogFlow = await insertHogFlow(transactionalFlow('api', 'active'))})+ afterEach(() => {+ jest.restoreAllMocks()+ })+it('queues one run on the email fast lane with the recipient and the data', async () => {const queueAppMetric = jest.spyOn(api['hogFunctionMonitoringService'], 'queueAppMetric')const queuedBefore = await outcomeCount('queued')@@ -2263,6 +2267,47 @@ describe('CDP API', () => {expect(mockQueueInvocations).not.toHaveBeenCalled()})+ it.each([+ [+ 'email sending is suspended for the project',+ 'sending_suspended',+ () =>+ jest+ .spyOn(api['teamWorkflowsConfigService'], 'getEmailSendingSuspension')+ .mockResolvedValueOnce('staff'),+ ],+ [+ "the transactional email's sending is paused",+ 'sending_paused',+ () =>+ jest+ .spyOn(api['hogFlowManager'], 'getHogFlow')+ .mockResolvedValueOnce({ ...hogFlow, email_sending_paused_at: Date.now() }),+ ],+ [+ 'the recipient is suppressed',+ 'recipient_suppressed',+ () => jest.spyOn(api['emailSuppressionService'], 'isSuppressed').mockResolvedValueOnce(true),+ ],+ ] as const)('refuses a send the email worker would skip because %s', async (_desc, code, refuse) => {+ refuse()+ const refusedBefore = await outcomeCount(code)+ const keyed = { ...body, idempotency_key: `key-${new UUIDT().toString()}` }+ const headers = { Authorization: `Bearer ${mintToken(team.id, hogFlow.id)}` }++ const refused = await send(hogFlow.id, headers, keyed)++ expect(refused.status).toEqual(422)+ expect(refused.body).toEqual({ error: code })+ expect(await outcomeCount(code)).toEqual(refusedBefore + 1)+ expect(mockQueueInvocations).not.toHaveBeenCalled()++ const retry = await send(hogFlow.id, headers, keyed)++ expect(retry.status).toEqual(202)+ expect(mockQueueInvocations).toHaveBeenCalledTimes(1)+ })+it.each([['a workflow with another trigger', 'schedule', 'active'],['a transactional email that is not active', 'api', 'draft'],@@ -2272,6 +2317,7 @@ describe('CDP API', () => {const res = await send(other.id, { Authorization: `Bearer ${mintToken(team.id, other.id)}` })expect(res.status).toEqual(400)+ expect(res.body).toEqual({ error: 'not_sendable' })expect(mockQueueInvocations).not.toHaveBeenCalled()})diff --git a/nodejs/src/cdp/cdp-api.ts b/nodejs/src/cdp/cdp-api.tsindex 5868cd1dcb56..9a168e1fbb32 100644--- a/nodejs/src/cdp/cdp-api.ts+++ b/nodejs/src/cdp/cdp-api.ts@@ -34,7 +34,7 @@ import {import { HogTransformerService, createHogTransformerService } from './hog-transformations/hog-transformer.service'import { RerunJobManager } from './rerun/rerun-job.manager'import { RerunRequest } from './rerun/rerun-job.types'-import { HogFlowAction } from './schema/hogflow'+import { HogFlow, HogFlowAction } from './schema/hogflow'import { BatchExportHogFunctionService, NotFoundError, ParseError } from './services/batch-export-hog-function.service'import type { CyclotronV2JobProducer } from './services/cyclotron-v2'import { HogExecutorAsyncService, HogExecutorExecuteAsyncOptions } from './services/hog-executor-async.service'@@ -62,7 +62,9 @@ import { InvocationResultsService } from './services/invocation-results.service'import { JobQueue } from './services/job-queue/job-queue.interface'import { GroupsManagerService } from './services/managers/groups-manager.service'import { HogFunctionManagerService } from './services/managers/hog-function-manager.service'+import { TeamWorkflowsConfigService } from './services/managers/team-workflows-config.service'import { EMAIL_QUEUE_PRIORITY } from './services/messaging/email-priority'+import { EmailSuppressionService } from './services/messaging/email-suppression.service'import { EmailTrackingService } from './services/messaging/email-tracking.service'import { EmailTrackingCodeSigner } from './services/messaging/helpers/tracking-code'import { RecipientTokensService } from './services/messaging/recipient-tokens.service'@@ -195,6 +197,8 @@ export class CdpApi {private stepResumeJwt: ScopedServiceJwtprivate transactionalEmailJwt: ScopedServiceJwtprivate transactionalEmailIdempotencyKeys: IdempotencyKeys+ private emailSuppressionService: EmailSuppressionService+ private teamWorkflowsConfigService: TeamWorkflowsConfigServiceprivate hogFunctionMonitoringService: HogFunctionMonitoringServiceprivate resumeBatchJwt: ScopedServiceJwt@@ -209,6 +213,8 @@ export class CdpApi {this.hogFunctionManager = services.hogFunctionManagerthis.hogFlowManager = services.hogFlowManagerthis.recipientTokensService = services.recipientTokensService+ this.emailSuppressionService = services.emailSuppressionService+ this.teamWorkflowsConfigService = services.teamWorkflowsConfigServicethis.hogExecutorAsync = services.hogExecutorAsyncthis.hogInputsService = services.hogInputsServicethis.hogFlowExecutor = services.hogFlowExecutor@@ -1174,7 +1180,7 @@ export class CdpApi {return reply('not_found', 404, { error: 'Workflow not found' })}if (hogFlow.trigger?.type !== 'api' || hogFlow.status !== 'active') {- return reply('not_sendable', 400, { error: 'Workflow is not an active transactional email' })+ return reply('not_sendable', 400, { error: 'not_sendable' })}const event: HogFunctionInvocationGlobals['event'] = {@@ -1226,12 +1232,12 @@ export class CdpApi {}}try {- const { isLimited } = await checkHogFlowQuotaLimits(hogFlow, team.id, this.deps.quotaLimiting)- if (isLimited) {+ const refusal = await this.refuseTransactionalEmail(team.id, hogFlow, to)+ if (refusal) {if (idempotencyKey) {await idempotencyKeys.release(team.id, idempotencyKey, invocation.id)}- return reply('quota_limited', 429, { error: 'quota_limited' })+ return reply(refusal.outcome, refusal.status, { error: refusal.outcome })}this.invocationResultsService.invocationResultsRowsService.queueLifecycleRow(invocation, 'running')try {@@ -1274,6 +1280,30 @@ export class CdpApi {}}+ // The email worker skips a send for these reasons too, but only after the caller has its 202. Checking them+ // here tells the caller that the email will not go out. The worker keeps its checks, because the state can+ // change while the job waits on the queue.+ private async refuseTransactionalEmail(+ teamId: number,+ hogFlow: HogFlow,+ to: string+ ): Promise<{ outcome: TransactionalEmailOutcome; status: number } | null> {+ if (await this.teamWorkflowsConfigService.getEmailSendingSuspension(teamId)) {+ return { outcome: 'sending_suspended', status: 422 }+ }+ if (hogFlow.email_sending_paused_at) {+ return { outcome: 'sending_paused', status: 422 }+ }+ if (await this.emailSuppressionService.isSuppressed(teamId, to)) {+ return { outcome: 'recipient_suppressed', status: 422 }+ }+ const { isLimited } = await checkHogFlowQuotaLimits(hogFlow, teamId, this.deps.quotaLimiting)+ if (isLimited) {+ return { outcome: 'quota_limited', status: 429 }+ }+ return null+ }+private verifyScopedWorkflowJwt(jwt: ScopedServiceJwt,req: ModifiedRequest,diff --git a/nodejs/src/cdp/services/hogflows/transactional-email.ts b/nodejs/src/cdp/services/hogflows/transactional-email.tsindex 8d0e9629c57c..7fae3a6489b5 100644--- a/nodejs/src/cdp/services/hogflows/transactional-email.ts+++ b/nodejs/src/cdp/services/hogflows/transactional-email.ts@@ -24,6 +24,9 @@ export type TransactionalEmailOutcome =| 'not_found'| 'not_sendable'| 'quota_limited'+ | 'sending_suspended'+ | 'sending_paused'+ | 'recipient_suppressed'| 'error'export const transactionalEmailRequestsTotal = new Counter({diff --git a/posthog/test/test_transactional_email.py b/posthog/test/test_transactional_email.pyindex f71a4d8c92d5..b8a336b70b7c 100644--- a/posthog/test/test_transactional_email.py+++ b/posthog/test/test_transactional_email.py@@ -85,14 +85,25 @@ class TestTransactionalEmail(SimpleTestCase):send_transactional_email("../different-endpoint", to="[email protected]")post.assert_not_called()- @parameterized.expand([(301,), (400,), (401,), (403,), (404,)])- def test_confirmed_rejection(self, status_code: int) -> None:+ @parameterized.expand(+ [+ (301, None, None),+ (400, None, None),+ (401, None, None),+ (403, None, None),+ (404, None, None),+ (422, {"type": "invalid_request", "code": "recipient_suppressed"}, "recipient_suppressed"),+ ]+ )+ def test_confirmed_rejection(self, status_code: int, body: dict | None, expected_code: str | None) -> None:+ response = Mock(status_code=status_code, json=Mock(return_value=body))with (- patch("posthog.transactional_email.requests.post", return_value=Mock(status_code=status_code)),+ patch("posthog.transactional_email.requests.post", return_value=response),self.assertRaises(TransactionalEmailRejected) as raised,):send_transactional_email(TEMPLATE_ID, to="[email protected]")assert str(raised.exception) == f"http_{status_code}"+ assert raised.exception.code == expected_code@parameterized.expand([(200,), (408,), (500,), (503,)])def test_unconfirmed_response_is_uncertain(self, status_code: int) -> None:diff --git a/posthog/transactional_email.py b/posthog/transactional_email.pyindex 215ae56c4471..18eadbaa1481 100644--- a/posthog/transactional_email.py+++ b/posthog/transactional_email.py@@ -18,6 +18,10 @@ class TransactionalEmailRetryableError(Exception):class TransactionalEmailRejected(Exception):"""The API rejected the email and requires caller intervention."""+ def __init__(self, message: str, *, code: str | None = None) -> None:+ super().__init__(message)+ self.code = code+class TransactionalEmailDeliveryUncertain(Exception):"""The API may have accepted the email; reconcile before retrying."""@@ -90,7 +94,7 @@ def send_transactional_email(retry_after = Noneraise TransactionalEmailRetryableError("rate_limited", retry_after=retry_after)if 300 <= response.status_code < 500 and response.status_code != 408:- raise TransactionalEmailRejected(f"http_{response.status_code}")+ raise TransactionalEmailRejected(f"http_{response.status_code}", code=_error_code(response))if response.status_code != 202:raise TransactionalEmailDeliveryUncertain(f"http_{response.status_code}")@@ -102,3 +106,12 @@ def send_transactional_email(if not isinstance(message_id, str) or not message_id:raise TransactionalEmailDeliveryUncertain("invalid_response")return message_id+++def _error_code(response: requests.Response) -> str | None:+ try:+ body = response.json()+ except ValueError:+ return None+ code = body.get("code") if isinstance(body, dict) else None+ return code if isinstance(code, str) else Nonediff --git a/products/messaging/backend/facade/contracts.py b/products/messaging/backend/facade/contracts.pyindex 3a83dd790b75..3cf3222bb4d7 100644--- a/products/messaging/backend/facade/contracts.py+++ b/products/messaging/backend/facade/contracts.py@@ -80,6 +80,18 @@ class IdempotencyKeyReused(TransactionalEmailError):code = "idempotency_key_reused"+class SendingSuspended(TransactionalEmailError):+ code = "sending_suspended"+++class SendingPaused(TransactionalEmailError):+ code = "sending_paused"+++class RecipientSuppressed(TransactionalEmailError):+ code = "recipient_suppressed"++class DeliveryUnavailable(Exception):"""The email pipeline did not accept the send. Safe to retry."""diff --git a/products/messaging/backend/presentation/views/message_templates.py b/products/messaging/backend/presentation/views/message_templates.pyindex 3573f6ffa6e1..07a1880036b2 100644--- a/products/messaging/backend/presentation/views/message_templates.py+++ b/products/messaging/backend/presentation/views/message_templates.py@@ -434,7 +434,14 @@ class SendTransactionalEmailResponseSerializer(serializers.Serializer):class TransactionalEmailRejected(exceptions.APIException):- STATUS_BY_CODE = {"quota_limited": 429, "idempotency_key_in_flight": 409, "idempotency_key_reused": 422}+ STATUS_BY_CODE = {+ "quota_limited": 429,+ "idempotency_key_in_flight": 409,+ "idempotency_key_reused": 422,+ "sending_suspended": 422,+ "sending_paused": 422,+ "recipient_suppressed": 422,+ }# The caller has to change the request. Without this the error handler reports a server error.default_type = "invalid_request"@@ -725,7 +732,11 @@ class MessageTemplatesViewSet(),404: OpenApiResponse(description="Template not found"),409: OpenApiResponse(description="A request with the same Idempotency-Key is still in progress"),- 422: OpenApiResponse(description="The Idempotency-Key was already used for a different request"),+ 422: OpenApiResponse(+ description="The email can't go out: the recipient is suppressed, sending is suspended for the "+ "project or paused for this transactional email, or the Idempotency-Key was already used for a "+ "different request"+ ),429: OpenApiResponse(description="Rate limited, or the project is over its email quota"),503: OpenApiResponse(description="The email pipeline did not confirm the send"),},diff --git a/products/messaging/backend/services/transactional_email.py b/products/messaging/backend/services/transactional_email.pyindex b322c5f2e3f7..de1fdf7b6c63 100644--- a/products/messaging/backend/services/transactional_email.py+++ b/products/messaging/backend/services/transactional_email.py@@ -2,7 +2,7 @@per send, so sends get the workflow's metrics, run history and versions."""import re-from typing import Any+from typing import TYPE_CHECKING, Anyfrom uuid import UUIDfrom django.db import transaction@@ -25,9 +25,13 @@ from products.messaging.backend.facade.contracts import (MarketingTemplate,QueuedEmail,QuotaLimited,+ RecipientSuppressed,SenderUnavailable,+ SendingPaused,+ SendingSuspended,TemplateNotSendable,TemplateNotTransactional,+ TransactionalEmailError,TransactionalEmailPaused,)from products.messaging.backend.models import MessageCategory, MessageTemplate@@ -42,8 +46,40 @@ from products.workflows.backend.facade.transactional_emails import (update_transactional_email_workflow,)+if TYPE_CHECKING:+ import requests+logger = structlog.get_logger(__name__)+# The CDP service names the reason it refused a send in the body's `error`. A retry of the same request gets the+# same refusal, so each of these reaches the caller as a 4xx and not as a retryable 503.+PIPELINE_REFUSALS: dict[str, tuple[type[TransactionalEmailError], str]] = {+ "quota_limited": (QuotaLimited, "The project is over its email quota."),+ "idempotency_key_in_flight": (+ IdempotencyKeyInFlight,+ "A request with this Idempotency-Key is still being processed. Retry in a few seconds.",+ ),+ "idempotency_key_reused": (+ IdempotencyKeyReused,+ "This Idempotency-Key was already used for a different request. Use a new key for a new email.",+ ),+ "sending_suspended": (+ SendingSuspended,+ "Email sending is suspended for this project. Check the Reputation tab in Workflows, or contact support.",+ ),+ "sending_paused": (+ SendingPaused,+ "Email sending is paused for this transactional email because of its bounce or complaint rate. "+ "Check the Reputation tab in Workflows.",+ ),+ "recipient_suppressed": (+ RecipientSuppressed,+ "The recipient is on this project's suppression list after a hard bounce or a spam complaint, so the "+ "email can't be sent. Remove the address from the suppression list if it can receive email again.",+ ),+ "not_sendable": (TransactionalEmailPaused, "The template's transactional email isn't active."),+}+IDEMPOTENCY_KEY_PATTERN = re.compile(r"[!-~]{1,255}")@@ -288,16 +324,10 @@ def send_transactional_email(raise DeliveryUnavailable() from eif response.status_code != 202:- if response.status_code == 429:- raise QuotaLimited("The project is over its email quota.")- if response.status_code == 409:- raise IdempotencyKeyInFlight(- "A request with this Idempotency-Key is still being processed. Retry in a few seconds."- )- if response.status_code == 422:- raise IdempotencyKeyReused(- "This Idempotency-Key was already used for a different request. Use a new key for a new email."- )+ refusal = PIPELINE_REFUSALS.get(_refusal_code(response) or "")+ if refusal is not None:+ error_class, message = refusal+ raise error_class(message)logger.error("transactional_email_queue_rejected",team_id=team.id,@@ -308,3 +338,12 @@ def send_transactional_email(body = response.json()return QueuedEmail(message_id=str(body["invocation_id"]), replayed=bool(body.get("replayed", False)))+++def _refusal_code(response: "requests.Response") -> str | None:+ try:+ body = response.json()+ except ValueError:+ return None+ code = body.get("error") if isinstance(body, dict) else None+ return code if isinstance(code, str) else Nonediff --git a/products/messaging/backend/tests/api/test_transactional_email.py b/products/messaging/backend/tests/api/test_transactional_email.pyindex 8479981cca75..8d06d53df075 100644--- a/products/messaging/backend/tests/api/test_transactional_email.py+++ b/products/messaging/backend/tests/api/test_transactional_email.py@@ -255,18 +255,24 @@ class TestTransactionalEmail(APIBaseTest):@parameterized.expand([- ("quota_limited", 429),- ("idempotency_key_in_flight", 409),- ("idempotency_key_reused", 422),+ ("quota_limited", 429, 429, "quota_limited"),+ ("idempotency_key_in_flight", 409, 409, "idempotency_key_in_flight"),+ ("idempotency_key_reused", 422, 422, "idempotency_key_reused"),+ ("sending_suspended", 422, 422, "sending_suspended"),+ ("sending_paused", 422, 422, "sending_paused"),+ ("recipient_suppressed", 422, 422, "recipient_suppressed"),+ ("not_sendable", 400, 400, "transactional_email_paused"),])- def test_reports_a_send_the_pipeline_refuses(self, mock_post, _flag, _capture, code, status_code):- mock_post.return_value = MagicMock(status_code=status_code, json=lambda: {"error": code})+ def test_reports_a_send_the_pipeline_refuses(+ self, mock_post, _flag, _capture, pipeline_code, pipeline_status, expected_status, expected_code+ ):+ mock_post.return_value = MagicMock(status_code=pipeline_status, json=lambda: {"error": pipeline_code})response = self.client.post(self.url, self.body, format="json", headers={"Idempotency-Key": "key-1"})- assert response.status_code == status_code- assert response.json()["code"] == code+ assert response.status_code == expected_status+ assert response.json()["code"] == expected_code@parameterized.expand([("send", "send/"), ("make_transactional", "make_transactional/")])def test_is_hidden_while_the_flag_is_off(self, mock_post, flag, _capture, _name, suffix):diff --git a/products/signals/backend/first_report_notification.py b/products/signals/backend/first_report_notification.pyindex b06a8be9f156..ad24b3c76a1a 100644--- a/products/signals/backend/first_report_notification.py+++ b/products/signals/backend/first_report_notification.py@@ -146,6 +146,10 @@ def deliver_first_report_email(team_id: int) -> None:message_data=notification.message_data,)except (TransactionalEmailRetryableError, TransactionalEmailRejected, ImproperlyConfigured) as error:+ if isinstance(error, TransactionalEmailRejected) and error.code == "recipient_suppressed":+ # The API refuses every later send to this address too, so a retry on each sweep can't succeed.+ record.update(send_started_at=None, cancelled_at=timezone.now(), last_error="recipient_suppressed")+ return# These failures occur before the workflow accepts a send, so they can safely retry.retry_after = error.retry_after if isinstance(error, TransactionalEmailRetryableError) else Nonerecord.update(diff --git a/products/signals/backend/test/test_first_report_notification.py b/products/signals/backend/test/test_first_report_notification.pyindex fa249e0d65cb..74641f48a9d0 100644--- a/products/signals/backend/test/test_first_report_notification.py+++ b/products/signals/backend/test/test_first_report_notification.py@@ -216,6 +216,17 @@ class TestFirstReportNotification(BaseTest):assert initial_send.kwargs["message_data"]["report_id"] == str(first.id)assert self.notification().queued_at is not None+ @time_machine.travel("2026-01-15T12:01:00Z", tick=False)+ def test_suppressed_recipient_cancels_the_notification(self) -> None:+ self.report()+ self.send.side_effect = TransactionalEmailRejected("http_422", code="recipient_suppressed")++ deliver_first_report_email(self.team.id)+ deliver_first_report_email(self.team.id)++ assert self.send.call_count == 1+ assert self.notification().cancelled_at is not None+def test_project_access_is_required_even_with_task_access(self) -> None:self.organization.available_product_features = [{"key": AvailableFeature.ACCESS_CONTROL, "name": AvailableFeature.ACCESS_CONTROL},