Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(messaging): drop transactional emails workflows refuses as unreachable (#124613)

Harley Alexander4ea4b1014307
diff --git a/posthog/email.py b/posthog/email.py
index b93a8b5c4e5b..8aed7fca2d0d 100644
--- a/posthog/email.py
+++ b/posthog/email.py
@@ -117,7 +117,7 @@ EMAIL_TASK_KWARGS = {
# cardinality (per-team volume lives in MessagingRecord); scraped via prometheus multiprocess.
EMAIL_SEND_COUNTER = Counter(
"posthog_email_send_total",
- "Email send attempts by outcome (sent|failed) and transport (smtp|http).",
+ "Email send attempts by outcome (sent|failed|undeliverable) and transport (smtp|http).",
labelnames=["outcome", "transport"],
)
@@ -224,6 +224,11 @@ WORKFLOWS_PIN_TTL_SECONDS = 24 * 60 * 60
# the workflow, or the recipient is on the suppression list. Customer.io sends these emails instead.
WORKFLOWS_NOT_QUEUED_422_CODES = frozenset({"sending_suspended", "sending_paused", "recipient_suppressed"})
+# 422 codes for an email the send API refused before queueing it because the address can't receive email: the syntax
+# is invalid or the domain has no mail servers. Every provider would hard bounce it, so the email is dropped. It is not
+# retried and Customer.io does not send it.
+WORKFLOWS_UNDELIVERABLE_422_CODES = frozenset({"recipient_unreachable"})
+
EMAIL_WORKFLOWS_FALLBACK_COUNTER = Counter(
"posthog_email_workflows_fallback_total",
"Emails that Workflows refused and Customer.io sent instead, by template.",
@@ -505,8 +510,11 @@ def _capture_email_event(capture: ScopedCapture, dest: dict[str, str], event: st
def _send_one(
capture: ScopedCapture, dest: dict[str, str], campaign_key: str, template_name: str, properties: dict
-) -> tuple[str, dict]:
- """Send one email and return the provider that sent it with the provider's response."""
+) -> tuple[str, dict] | None:
+ """Send one email and return the provider that sent it with the provider's response.
+
+ Returns None when Workflows refused the recipient as undeliverable, so no provider sent the email.
+ """
idempotency_key = _workflows_idempotency_key(campaign_key, dest["raw_email"])
# After an attempt Workflows may have queued, a retry skips the flag and never falls back. Otherwise a flag
# change or a different error on the retry would send the same email again through Customer.io.
@@ -519,6 +527,19 @@ def _send_one(
_pin_to_workflows(idempotency_key, workflows_template_id)
return "workflows", response
except WorkflowsEmailError as err:
+ if err.status_code == 422 and err.code in WORKFLOWS_UNDELIVERABLE_422_CODES:
+ # The send API replays an idempotency key before it checks the recipient, so this refusal also
+ # means that no earlier attempt queued the email. The pin keeps a retry of the batch on Workflows, so a
+ # flag change cannot send this recipient through Customer.io, where the email would hard bounce.
+ _pin_to_workflows(idempotency_key, workflows_template_id)
+ logger.info("email_workflows_recipient_undeliverable", template_name=template_name)
+ _capture_email_event(
+ capture,
+ dest,
+ "transactional email workflows rejected",
+ {"template_name": template_name, "campaign_key": campaign_key, "error": str(err)[:500]},
+ )
+ return None
if pinned_template_id or not _workflows_error_allows_fallback(err):
_pin_to_workflows(idempotency_key, workflows_template_id)
raise
@@ -561,6 +582,7 @@ def _send_via_http_batch(
) -> None:
sent_count = 0
already_sent_count = 0 # delivered in a prior run — not a failure if the batch later aborts
+ undeliverable_count = 0
dest: dict[str, str] | None = None
try:
for dest in to:
@@ -574,7 +596,13 @@ def _send_via_http_batch(
already_sent_count += 1
continue
- provider, provider_response = _send_one(capture, dest, campaign_key, template_name, properties)
+ sent = _send_one(capture, dest, campaign_key, template_name, properties)
+ if sent is None:
+ # sent_at stays unset, so raise_if_delivery_rejected tells a synchronous caller nothing went out.
+ EMAIL_SEND_COUNTER.labels(outcome="undeliverable", transport="http").inc()
+ undeliverable_count += 1
+ continue
+ provider, provider_response = sent
# Mark delivery before any non-essential step so a delivered email is never
# recorded as rejected (raise_if_delivery_rejected keys off sent_at). Analytics
@@ -617,7 +645,9 @@ def _send_via_http_batch(
)
# Count every recipient that did not get through (the failing one + any not yet attempted),
# so `failed` shares the per-recipient unit with `sent` instead of registering once per batch.
- EMAIL_SEND_COUNTER.labels(outcome="failed", transport="http").inc(len(to) - already_sent_count - sent_count)
+ EMAIL_SEND_COUNTER.labels(outcome="failed", transport="http").inc(
+ len(to) - already_sent_count - sent_count - undeliverable_count
+ )
if retryable:
# The retry sends the same idempotency key, so Workflows returns an email it already queued instead of
# sending a second one, and recipients already marked sent are skipped.
diff --git a/posthog/test/test_email.py b/posthog/test/test_email.py
index bdd324af7634..b1331cc9e04f 100644
--- a/posthog/test/test_email.py
+++ b/posthog/test/test_email.py
@@ -483,6 +483,14 @@ class TestEmail(BaseTest):
"customer_io",
["transactional email workflows rejected", "transactional email triggered"],
),
+ (
+ "workflows_refuses_unreachable_recipient_drops_email",
+ True,
+ [(422, {"type": "invalid_request", "code": "recipient_unreachable"})],
+ ["workflows"],
+ None,
+ ["transactional email workflows rejected"],
+ ),
(
"workflows_key_already_used_sends_nothing",
True,
@@ -591,7 +599,7 @@ class TestEmail(BaseTest):
use_http=True,
)
message.add_recipient("[email protected]", "Test", distinct_id="user-1")
- if sent_by is None:
+ if "transactional email failed" in expected_events:
# Workflows may have queued the email, so the error reaches Celery, which retries with the same key.
with self.assertRaises(WorkflowsEmailError):
message.send(send_async=False, retry=False)
@@ -717,6 +725,45 @@ class TestEmail(BaseTest):
]
self.assertEqual(providers, expected_providers)
+ @patch("posthog.email.ph_scoped_capture")
+ @patch("posthog.email.posthoganalytics.feature_enabled")
+ @patch("posthog.email.requests.post")
+ def test_batch_retry_keeps_an_unreachable_recipient_off_customer_io(
+ self, mock_post: MagicMock, mock_feature_enabled: MagicMock, _mock_capture: MagicMock
+ ) -> None:
+ mock_feature_enabled.return_value = True
+ unreachable = MagicMock(status_code=422, text="error")
+ unreachable.json.return_value = {"type": "invalid_request", "code": "recipient_unreachable"}
+ unavailable = MagicMock(status_code=503, text="error")
+ unavailable.json.return_value = {"code": "delivery_unavailable"}
+ accepted = MagicMock(status_code=202, json=MagicMock(return_value={"message_id": "m1"}))
+ mock_post.side_effect = [unreachable, unavailable, unreachable, accepted]
+
+ with (
+ override_instance_config("EMAIL_HOST", "localhost"),
+ self.settings(
+ CUSTOMER_IO_API_KEY="cio-key",
+ POSTHOG_TRANSACTIONAL_EMAIL_API_KEY="phs_test",
+ POSTHOG_HOST="https://us.posthog.com",
+ ),
+ patch.dict("posthog.email.WORKFLOWS_TEMPLATE_ID_MAP", {"password_reset": "tmpl-1"}),
+ ):
+ message = EmailMessage(
+ campaign_key="unreachable_batch", subject="Test subject", template_name="password_reset", use_http=True
+ )
+ message.add_recipient("[email protected]")
+ message.add_recipient("[email protected]")
+ with self.assertRaises(WorkflowsEmailError):
+ message.send(send_async=False, retry=False)
+ mock_feature_enabled.return_value = False
+ message.send(send_async=False, retry=False)
+
+ self.assertTrue(all(call.args[0].startswith("https://us.posthog.com") for call in mock_post.call_args_list))
+ self.assertEqual(
+ [call.kwargs["json"]["to"] for call in mock_post.call_args_list],
+ )
+
@patch("posthog.email.ph_scoped_capture")
@patch("posthog.email.posthoganalytics.feature_enabled")
@patch("posthog.email.requests.post")