Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(messaging): refuse transactional sends the email worker would skip (#123019)

Harley Alexander02ce58f85782
diff --git a/nodejs/src/cdp/cdp-api.serial.test.ts b/nodejs/src/cdp/cdp-api.serial.test.ts
index 6a5914f4fcc1..1fcc77b91803 100644
--- a/nodejs/src/cdp/cdp-api.serial.test.ts
+++ b/nodejs/src/cdp/cdp-api.serial.test.ts
@@ -2146,6 +2146,10 @@ describe('CDP API', () => {
hogFlow = await insertHogFlow(transactionalFlow('api', 'active'))
})
+ afterEach(() => {
+ jest.restoreAllMocks()
+ })
+
it('queues one run on the email fast lane with the recipient and the data', async () => {
const queueAppMetric = jest.spyOn(api['hogFunctionMonitoringService'], 'queueAppMetric')
const queuedBefore = await outcomeCount('queued')
@@ -2263,6 +2267,47 @@ describe('CDP API', () => {
expect(mockQueueInvocations).not.toHaveBeenCalled()
})
+ it.each([
+ [
+ 'email sending is suspended for the project',
+ 'sending_suspended',
+ () =>
+ jest
+ .spyOn(api['teamWorkflowsConfigService'], 'getEmailSendingSuspension')
+ .mockResolvedValueOnce('staff'),
+ ],
+ [
+ "the transactional email's sending is paused",
+ 'sending_paused',
+ () =>
+ jest
+ .spyOn(api['hogFlowManager'], 'getHogFlow')
+ .mockResolvedValueOnce({ ...hogFlow, email_sending_paused_at: Date.now() }),
+ ],
+ [
+ 'the recipient is suppressed',
+ 'recipient_suppressed',
+ () => jest.spyOn(api['emailSuppressionService'], 'isSuppressed').mockResolvedValueOnce(true),
+ ],
+ ] as const)('refuses a send the email worker would skip because %s', async (_desc, code, refuse) => {
+ refuse()
+ const refusedBefore = await outcomeCount(code)
+ const keyed = { ...body, idempotency_key: `key-${new UUIDT().toString()}` }
+ const headers = { Authorization: `Bearer ${mintToken(team.id, hogFlow.id)}` }
+
+ const refused = await send(hogFlow.id, headers, keyed)
+
+ expect(refused.status).toEqual(422)
+ expect(refused.body).toEqual({ error: code })
+ expect(await outcomeCount(code)).toEqual(refusedBefore + 1)
+ expect(mockQueueInvocations).not.toHaveBeenCalled()
+
+ const retry = await send(hogFlow.id, headers, keyed)
+
+ expect(retry.status).toEqual(202)
+ expect(mockQueueInvocations).toHaveBeenCalledTimes(1)
+ })
+
it.each([
['a workflow with another trigger', 'schedule', 'active'],
['a transactional email that is not active', 'api', 'draft'],
@@ -2272,6 +2317,7 @@ describe('CDP API', () => {
const res = await send(other.id, { Authorization: `Bearer ${mintToken(team.id, other.id)}` })
expect(res.status).toEqual(400)
+ expect(res.body).toEqual({ error: 'not_sendable' })
expect(mockQueueInvocations).not.toHaveBeenCalled()
})
diff --git a/nodejs/src/cdp/cdp-api.ts b/nodejs/src/cdp/cdp-api.ts
index 5868cd1dcb56..9a168e1fbb32 100644
--- a/nodejs/src/cdp/cdp-api.ts
+++ b/nodejs/src/cdp/cdp-api.ts
@@ -34,7 +34,7 @@ import {
import { HogTransformerService, createHogTransformerService } from './hog-transformations/hog-transformer.service'
import { RerunJobManager } from './rerun/rerun-job.manager'
import { RerunRequest } from './rerun/rerun-job.types'
-import { HogFlowAction } from './schema/hogflow'
+import { HogFlow, HogFlowAction } from './schema/hogflow'
import { BatchExportHogFunctionService, NotFoundError, ParseError } from './services/batch-export-hog-function.service'
import type { CyclotronV2JobProducer } from './services/cyclotron-v2'
import { HogExecutorAsyncService, HogExecutorExecuteAsyncOptions } from './services/hog-executor-async.service'
@@ -62,7 +62,9 @@ import { InvocationResultsService } from './services/invocation-results.service'
import { JobQueue } from './services/job-queue/job-queue.interface'
import { GroupsManagerService } from './services/managers/groups-manager.service'
import { HogFunctionManagerService } from './services/managers/hog-function-manager.service'
+import { TeamWorkflowsConfigService } from './services/managers/team-workflows-config.service'
import { EMAIL_QUEUE_PRIORITY } from './services/messaging/email-priority'
+import { EmailSuppressionService } from './services/messaging/email-suppression.service'
import { EmailTrackingService } from './services/messaging/email-tracking.service'
import { EmailTrackingCodeSigner } from './services/messaging/helpers/tracking-code'
import { RecipientTokensService } from './services/messaging/recipient-tokens.service'
@@ -195,6 +197,8 @@ export class CdpApi {
private stepResumeJwt: ScopedServiceJwt
private transactionalEmailJwt: ScopedServiceJwt
private transactionalEmailIdempotencyKeys: IdempotencyKeys
+ private emailSuppressionService: EmailSuppressionService
+ private teamWorkflowsConfigService: TeamWorkflowsConfigService
private hogFunctionMonitoringService: HogFunctionMonitoringService
private resumeBatchJwt: ScopedServiceJwt
@@ -209,6 +213,8 @@ export class CdpApi {
this.hogFunctionManager = services.hogFunctionManager
this.hogFlowManager = services.hogFlowManager
this.recipientTokensService = services.recipientTokensService
+ this.emailSuppressionService = services.emailSuppressionService
+ this.teamWorkflowsConfigService = services.teamWorkflowsConfigService
this.hogExecutorAsync = services.hogExecutorAsync
this.hogInputsService = services.hogInputsService
this.hogFlowExecutor = services.hogFlowExecutor
@@ -1174,7 +1180,7 @@ export class CdpApi {
return reply('not_found', 404, { error: 'Workflow not found' })
}
if (hogFlow.trigger?.type !== 'api' || hogFlow.status !== 'active') {
- return reply('not_sendable', 400, { error: 'Workflow is not an active transactional email' })
+ return reply('not_sendable', 400, { error: 'not_sendable' })
}
const event: HogFunctionInvocationGlobals['event'] = {
@@ -1226,12 +1232,12 @@ export class CdpApi {
}
}
try {
- const { isLimited } = await checkHogFlowQuotaLimits(hogFlow, team.id, this.deps.quotaLimiting)
- if (isLimited) {
+ const refusal = await this.refuseTransactionalEmail(team.id, hogFlow, to)
+ if (refusal) {
if (idempotencyKey) {
await idempotencyKeys.release(team.id, idempotencyKey, invocation.id)
}
- return reply('quota_limited', 429, { error: 'quota_limited' })
+ return reply(refusal.outcome, refusal.status, { error: refusal.outcome })
}
this.invocationResultsService.invocationResultsRowsService.queueLifecycleRow(invocation, 'running')
try {
@@ -1274,6 +1280,30 @@ export class CdpApi {
}
}
+ // The email worker skips a send for these reasons too, but only after the caller has its 202. Checking them
+ // here tells the caller that the email will not go out. The worker keeps its checks, because the state can
+ // change while the job waits on the queue.
+ private async refuseTransactionalEmail(
+ teamId: number,
+ hogFlow: HogFlow,
+ to: string
+ ): Promise<{ outcome: TransactionalEmailOutcome; status: number } | null> {
+ if (await this.teamWorkflowsConfigService.getEmailSendingSuspension(teamId)) {
+ return { outcome: 'sending_suspended', status: 422 }
+ }
+ if (hogFlow.email_sending_paused_at) {
+ return { outcome: 'sending_paused', status: 422 }
+ }
+ if (await this.emailSuppressionService.isSuppressed(teamId, to)) {
+ return { outcome: 'recipient_suppressed', status: 422 }
+ }
+ const { isLimited } = await checkHogFlowQuotaLimits(hogFlow, teamId, this.deps.quotaLimiting)
+ if (isLimited) {
+ return { outcome: 'quota_limited', status: 429 }
+ }
+ return null
+ }
+
private verifyScopedWorkflowJwt(
jwt: ScopedServiceJwt,
req: ModifiedRequest,
diff --git a/nodejs/src/cdp/services/hogflows/transactional-email.ts b/nodejs/src/cdp/services/hogflows/transactional-email.ts
index 8d0e9629c57c..7fae3a6489b5 100644
--- a/nodejs/src/cdp/services/hogflows/transactional-email.ts
+++ b/nodejs/src/cdp/services/hogflows/transactional-email.ts
@@ -24,6 +24,9 @@ export type TransactionalEmailOutcome =
| 'not_found'
| 'not_sendable'
| 'quota_limited'
+ | 'sending_suspended'
+ | 'sending_paused'
+ | 'recipient_suppressed'
| 'error'
export const transactionalEmailRequestsTotal = new Counter({
diff --git a/posthog/test/test_transactional_email.py b/posthog/test/test_transactional_email.py
index f71a4d8c92d5..b8a336b70b7c 100644
--- a/posthog/test/test_transactional_email.py
+++ b/posthog/test/test_transactional_email.py
@@ -85,14 +85,25 @@ class TestTransactionalEmail(SimpleTestCase):
send_transactional_email("../different-endpoint", to="[email protected]")
post.assert_not_called()
- @parameterized.expand([(301,), (400,), (401,), (403,), (404,)])
- def test_confirmed_rejection(self, status_code: int) -> None:
+ @parameterized.expand(
+ [
+ (301, None, None),
+ (400, None, None),
+ (401, None, None),
+ (403, None, None),
+ (404, None, None),
+ (422, {"type": "invalid_request", "code": "recipient_suppressed"}, "recipient_suppressed"),
+ ]
+ )
+ def test_confirmed_rejection(self, status_code: int, body: dict | None, expected_code: str | None) -> None:
+ response = Mock(status_code=status_code, json=Mock(return_value=body))
with (
- patch("posthog.transactional_email.requests.post", return_value=Mock(status_code=status_code)),
+ patch("posthog.transactional_email.requests.post", return_value=response),
self.assertRaises(TransactionalEmailRejected) as raised,
):
send_transactional_email(TEMPLATE_ID, to="[email protected]")
assert str(raised.exception) == f"http_{status_code}"
+ assert raised.exception.code == expected_code
@parameterized.expand([(200,), (408,), (500,), (503,)])
def test_unconfirmed_response_is_uncertain(self, status_code: int) -> None:
diff --git a/posthog/transactional_email.py b/posthog/transactional_email.py
index 215ae56c4471..18eadbaa1481 100644
--- a/posthog/transactional_email.py
+++ b/posthog/transactional_email.py
@@ -18,6 +18,10 @@ class TransactionalEmailRetryableError(Exception):
class TransactionalEmailRejected(Exception):
"""The API rejected the email and requires caller intervention."""
+ def __init__(self, message: str, *, code: str | None = None) -> None:
+ super().__init__(message)
+ self.code = code
+
class TransactionalEmailDeliveryUncertain(Exception):
"""The API may have accepted the email; reconcile before retrying."""
@@ -90,7 +94,7 @@ def send_transactional_email(
retry_after = None
raise TransactionalEmailRetryableError("rate_limited", retry_after=retry_after)
if 300 <= response.status_code < 500 and response.status_code != 408:
- raise TransactionalEmailRejected(f"http_{response.status_code}")
+ raise TransactionalEmailRejected(f"http_{response.status_code}", code=_error_code(response))
if response.status_code != 202:
raise TransactionalEmailDeliveryUncertain(f"http_{response.status_code}")
@@ -102,3 +106,12 @@ def send_transactional_email(
if not isinstance(message_id, str) or not message_id:
raise TransactionalEmailDeliveryUncertain("invalid_response")
return message_id
+
+
+def _error_code(response: requests.Response) -> str | None:
+ try:
+ body = response.json()
+ except ValueError:
+ return None
+ code = body.get("code") if isinstance(body, dict) else None
+ return code if isinstance(code, str) else None
diff --git a/products/messaging/backend/facade/contracts.py b/products/messaging/backend/facade/contracts.py
index 3a83dd790b75..3cf3222bb4d7 100644
--- a/products/messaging/backend/facade/contracts.py
+++ b/products/messaging/backend/facade/contracts.py
@@ -80,6 +80,18 @@ class IdempotencyKeyReused(TransactionalEmailError):
code = "idempotency_key_reused"
+class SendingSuspended(TransactionalEmailError):
+ code = "sending_suspended"
+
+
+class SendingPaused(TransactionalEmailError):
+ code = "sending_paused"
+
+
+class RecipientSuppressed(TransactionalEmailError):
+ code = "recipient_suppressed"
+
+
class DeliveryUnavailable(Exception):
"""The email pipeline did not accept the send. Safe to retry."""
diff --git a/products/messaging/backend/presentation/views/message_templates.py b/products/messaging/backend/presentation/views/message_templates.py
index 3573f6ffa6e1..07a1880036b2 100644
--- a/products/messaging/backend/presentation/views/message_templates.py
+++ b/products/messaging/backend/presentation/views/message_templates.py
@@ -434,7 +434,14 @@ class SendTransactionalEmailResponseSerializer(serializers.Serializer):
class TransactionalEmailRejected(exceptions.APIException):
- STATUS_BY_CODE = {"quota_limited": 429, "idempotency_key_in_flight": 409, "idempotency_key_reused": 422}
+ STATUS_BY_CODE = {
+ "quota_limited": 429,
+ "idempotency_key_in_flight": 409,
+ "idempotency_key_reused": 422,
+ "sending_suspended": 422,
+ "sending_paused": 422,
+ "recipient_suppressed": 422,
+ }
# The caller has to change the request. Without this the error handler reports a server error.
default_type = "invalid_request"
@@ -725,7 +732,11 @@ class MessageTemplatesViewSet(
),
404: OpenApiResponse(description="Template not found"),
409: OpenApiResponse(description="A request with the same Idempotency-Key is still in progress"),
- 422: OpenApiResponse(description="The Idempotency-Key was already used for a different request"),
+ 422: OpenApiResponse(
+ description="The email can't go out: the recipient is suppressed, sending is suspended for the "
+ "project or paused for this transactional email, or the Idempotency-Key was already used for a "
+ "different request"
+ ),
429: OpenApiResponse(description="Rate limited, or the project is over its email quota"),
503: OpenApiResponse(description="The email pipeline did not confirm the send"),
},
diff --git a/products/messaging/backend/services/transactional_email.py b/products/messaging/backend/services/transactional_email.py
index b322c5f2e3f7..de1fdf7b6c63 100644
--- a/products/messaging/backend/services/transactional_email.py
+++ b/products/messaging/backend/services/transactional_email.py
@@ -2,7 +2,7 @@
per send, so sends get the workflow's metrics, run history and versions."""
import re
-from typing import Any
+from typing import TYPE_CHECKING, Any
from uuid import UUID
from django.db import transaction
@@ -25,9 +25,13 @@ from products.messaging.backend.facade.contracts import (
MarketingTemplate,
QueuedEmail,
QuotaLimited,
+ RecipientSuppressed,
SenderUnavailable,
+ SendingPaused,
+ SendingSuspended,
TemplateNotSendable,
TemplateNotTransactional,
+ TransactionalEmailError,
TransactionalEmailPaused,
)
from products.messaging.backend.models import MessageCategory, MessageTemplate
@@ -42,8 +46,40 @@ from products.workflows.backend.facade.transactional_emails import (
update_transactional_email_workflow,
)
+if TYPE_CHECKING:
+ import requests
+
logger = structlog.get_logger(__name__)
+# The CDP service names the reason it refused a send in the body's `error`. A retry of the same request gets the
+# same refusal, so each of these reaches the caller as a 4xx and not as a retryable 503.
+PIPELINE_REFUSALS: dict[str, tuple[type[TransactionalEmailError], str]] = {
+ "quota_limited": (QuotaLimited, "The project is over its email quota."),
+ "idempotency_key_in_flight": (
+ IdempotencyKeyInFlight,
+ "A request with this Idempotency-Key is still being processed. Retry in a few seconds.",
+ ),
+ "idempotency_key_reused": (
+ IdempotencyKeyReused,
+ "This Idempotency-Key was already used for a different request. Use a new key for a new email.",
+ ),
+ "sending_suspended": (
+ SendingSuspended,
+ "Email sending is suspended for this project. Check the Reputation tab in Workflows, or contact support.",
+ ),
+ "sending_paused": (
+ SendingPaused,
+ "Email sending is paused for this transactional email because of its bounce or complaint rate. "
+ "Check the Reputation tab in Workflows.",
+ ),
+ "recipient_suppressed": (
+ RecipientSuppressed,
+ "The recipient is on this project's suppression list after a hard bounce or a spam complaint, so the "
+ "email can't be sent. Remove the address from the suppression list if it can receive email again.",
+ ),
+ "not_sendable": (TransactionalEmailPaused, "The template's transactional email isn't active."),
+}
+
IDEMPOTENCY_KEY_PATTERN = re.compile(r"[!-~]{1,255}")
@@ -288,16 +324,10 @@ def send_transactional_email(
raise DeliveryUnavailable() from e
if response.status_code != 202:
- if response.status_code == 429:
- raise QuotaLimited("The project is over its email quota.")
- if response.status_code == 409:
- raise IdempotencyKeyInFlight(
- "A request with this Idempotency-Key is still being processed. Retry in a few seconds."
- )
- if response.status_code == 422:
- raise IdempotencyKeyReused(
- "This Idempotency-Key was already used for a different request. Use a new key for a new email."
- )
+ refusal = PIPELINE_REFUSALS.get(_refusal_code(response) or "")
+ if refusal is not None:
+ error_class, message = refusal
+ raise error_class(message)
logger.error(
"transactional_email_queue_rejected",
team_id=team.id,
@@ -308,3 +338,12 @@ def send_transactional_email(
body = response.json()
return QueuedEmail(message_id=str(body["invocation_id"]), replayed=bool(body.get("replayed", False)))
+
+
+def _refusal_code(response: "requests.Response") -> str | None:
+ try:
+ body = response.json()
+ except ValueError:
+ return None
+ code = body.get("error") if isinstance(body, dict) else None
+ return code if isinstance(code, str) else None
diff --git a/products/messaging/backend/tests/api/test_transactional_email.py b/products/messaging/backend/tests/api/test_transactional_email.py
index 8479981cca75..8d06d53df075 100644
--- a/products/messaging/backend/tests/api/test_transactional_email.py
+++ b/products/messaging/backend/tests/api/test_transactional_email.py
@@ -255,18 +255,24 @@ class TestTransactionalEmail(APIBaseTest):
@parameterized.expand(
[
- ("quota_limited", 429),
- ("idempotency_key_in_flight", 409),
- ("idempotency_key_reused", 422),
+ ("quota_limited", 429, 429, "quota_limited"),
+ ("idempotency_key_in_flight", 409, 409, "idempotency_key_in_flight"),
+ ("idempotency_key_reused", 422, 422, "idempotency_key_reused"),
+ ("sending_suspended", 422, 422, "sending_suspended"),
+ ("sending_paused", 422, 422, "sending_paused"),
+ ("recipient_suppressed", 422, 422, "recipient_suppressed"),
+ ("not_sendable", 400, 400, "transactional_email_paused"),
]
)
- def test_reports_a_send_the_pipeline_refuses(self, mock_post, _flag, _capture, code, status_code):
- mock_post.return_value = MagicMock(status_code=status_code, json=lambda: {"error": code})
+ def test_reports_a_send_the_pipeline_refuses(
+ self, mock_post, _flag, _capture, pipeline_code, pipeline_status, expected_status, expected_code
+ ):
+ mock_post.return_value = MagicMock(status_code=pipeline_status, json=lambda: {"error": pipeline_code})
response = self.client.post(self.url, self.body, format="json", headers={"Idempotency-Key": "key-1"})
- assert response.status_code == status_code
- assert response.json()["code"] == code
+ assert response.status_code == expected_status
+ assert response.json()["code"] == expected_code
@parameterized.expand([("send", "send/"), ("make_transactional", "make_transactional/")])
def test_is_hidden_while_the_flag_is_off(self, mock_post, flag, _capture, _name, suffix):
diff --git a/products/signals/backend/first_report_notification.py b/products/signals/backend/first_report_notification.py
index b06a8be9f156..ad24b3c76a1a 100644
--- a/products/signals/backend/first_report_notification.py
+++ b/products/signals/backend/first_report_notification.py
@@ -146,6 +146,10 @@ def deliver_first_report_email(team_id: int) -> None:
message_data=notification.message_data,
)
except (TransactionalEmailRetryableError, TransactionalEmailRejected, ImproperlyConfigured) as error:
+ if isinstance(error, TransactionalEmailRejected) and error.code == "recipient_suppressed":
+ # The API refuses every later send to this address too, so a retry on each sweep can't succeed.
+ record.update(send_started_at=None, cancelled_at=timezone.now(), last_error="recipient_suppressed")
+ return
# These failures occur before the workflow accepts a send, so they can safely retry.
retry_after = error.retry_after if isinstance(error, TransactionalEmailRetryableError) else None
record.update(
diff --git a/products/signals/backend/test/test_first_report_notification.py b/products/signals/backend/test/test_first_report_notification.py
index fa249e0d65cb..74641f48a9d0 100644
--- a/products/signals/backend/test/test_first_report_notification.py
+++ b/products/signals/backend/test/test_first_report_notification.py
@@ -216,6 +216,17 @@ class TestFirstReportNotification(BaseTest):
assert initial_send.kwargs["message_data"]["report_id"] == str(first.id)
assert self.notification().queued_at is not None
+ @time_machine.travel("2026-01-15T12:01:00Z", tick=False)
+ def test_suppressed_recipient_cancels_the_notification(self) -> None:
+ self.report()
+ self.send.side_effect = TransactionalEmailRejected("http_422", code="recipient_suppressed")
+
+ deliver_first_report_email(self.team.id)
+ deliver_first_report_email(self.team.id)
+
+ assert self.send.call_count == 1
+ assert self.notification().cancelled_at is not None
+
def test_project_access_is_required_even_with_task_access(self) -> None:
self.organization.available_product_features = [
{"key": AvailableFeature.ACCESS_CONTROL, "name": AvailableFeature.ACCESS_CONTROL},