:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(messaging): drop transactional emails workflows refuses as unreachable (#124613)
Harley Alexander4ea4b1014307
diff --git a/posthog/email.py b/posthog/email.pyindex b93a8b5c4e5b..8aed7fca2d0d 100644--- a/posthog/email.py+++ b/posthog/email.py@@ -117,7 +117,7 @@ EMAIL_TASK_KWARGS = {# cardinality (per-team volume lives in MessagingRecord); scraped via prometheus multiprocess.EMAIL_SEND_COUNTER = Counter("posthog_email_send_total",- "Email send attempts by outcome (sent|failed) and transport (smtp|http).",+ "Email send attempts by outcome (sent|failed|undeliverable) and transport (smtp|http).",labelnames=["outcome", "transport"],)@@ -224,6 +224,11 @@ WORKFLOWS_PIN_TTL_SECONDS = 24 * 60 * 60# the workflow, or the recipient is on the suppression list. Customer.io sends these emails instead.WORKFLOWS_NOT_QUEUED_422_CODES = frozenset({"sending_suspended", "sending_paused", "recipient_suppressed"})+# 422 codes for an email the send API refused before queueing it because the address can't receive email: the syntax+# is invalid or the domain has no mail servers. Every provider would hard bounce it, so the email is dropped. It is not+# retried and Customer.io does not send it.+WORKFLOWS_UNDELIVERABLE_422_CODES = frozenset({"recipient_unreachable"})+EMAIL_WORKFLOWS_FALLBACK_COUNTER = Counter("posthog_email_workflows_fallback_total","Emails that Workflows refused and Customer.io sent instead, by template.",@@ -505,8 +510,11 @@ def _capture_email_event(capture: ScopedCapture, dest: dict[str, str], event: stdef _send_one(capture: ScopedCapture, dest: dict[str, str], campaign_key: str, template_name: str, properties: dict-) -> tuple[str, dict]:- """Send one email and return the provider that sent it with the provider's response."""+) -> tuple[str, dict] | None:+ """Send one email and return the provider that sent it with the provider's response.++ Returns None when Workflows refused the recipient as undeliverable, so no provider sent the email.+ """idempotency_key = _workflows_idempotency_key(campaign_key, dest["raw_email"])# After an attempt Workflows may have queued, a retry skips the flag and never falls back. Otherwise a flag# change or a different error on the retry would send the same email again through Customer.io.@@ -519,6 +527,19 @@ def _send_one(_pin_to_workflows(idempotency_key, workflows_template_id)return "workflows", responseexcept WorkflowsEmailError as err:+ if err.status_code == 422 and err.code in WORKFLOWS_UNDELIVERABLE_422_CODES:+ # The send API replays an idempotency key before it checks the recipient, so this refusal also+ # means that no earlier attempt queued the email. The pin keeps a retry of the batch on Workflows, so a+ # flag change cannot send this recipient through Customer.io, where the email would hard bounce.+ _pin_to_workflows(idempotency_key, workflows_template_id)+ logger.info("email_workflows_recipient_undeliverable", template_name=template_name)+ _capture_email_event(+ capture,+ dest,+ "transactional email workflows rejected",+ {"template_name": template_name, "campaign_key": campaign_key, "error": str(err)[:500]},+ )+ return Noneif pinned_template_id or not _workflows_error_allows_fallback(err):_pin_to_workflows(idempotency_key, workflows_template_id)raise@@ -561,6 +582,7 @@ def _send_via_http_batch() -> None:sent_count = 0already_sent_count = 0 # delivered in a prior run — not a failure if the batch later aborts+ undeliverable_count = 0dest: dict[str, str] | None = Nonetry:for dest in to:@@ -574,7 +596,13 @@ def _send_via_http_batch(already_sent_count += 1continue- provider, provider_response = _send_one(capture, dest, campaign_key, template_name, properties)+ sent = _send_one(capture, dest, campaign_key, template_name, properties)+ if sent is None:+ # sent_at stays unset, so raise_if_delivery_rejected tells a synchronous caller nothing went out.+ EMAIL_SEND_COUNTER.labels(outcome="undeliverable", transport="http").inc()+ undeliverable_count += 1+ continue+ provider, provider_response = sent# Mark delivery before any non-essential step so a delivered email is never# recorded as rejected (raise_if_delivery_rejected keys off sent_at). Analytics@@ -617,7 +645,9 @@ def _send_via_http_batch()# Count every recipient that did not get through (the failing one + any not yet attempted),# so `failed` shares the per-recipient unit with `sent` instead of registering once per batch.- EMAIL_SEND_COUNTER.labels(outcome="failed", transport="http").inc(len(to) - already_sent_count - sent_count)+ EMAIL_SEND_COUNTER.labels(outcome="failed", transport="http").inc(+ len(to) - already_sent_count - sent_count - undeliverable_count+ )if retryable:# The retry sends the same idempotency key, so Workflows returns an email it already queued instead of# sending a second one, and recipients already marked sent are skipped.diff --git a/posthog/test/test_email.py b/posthog/test/test_email.pyindex bdd324af7634..b1331cc9e04f 100644--- a/posthog/test/test_email.py+++ b/posthog/test/test_email.py@@ -483,6 +483,14 @@ class TestEmail(BaseTest):"customer_io",["transactional email workflows rejected", "transactional email triggered"],),+ (+ "workflows_refuses_unreachable_recipient_drops_email",+ True,+ [(422, {"type": "invalid_request", "code": "recipient_unreachable"})],+ ["workflows"],+ None,+ ["transactional email workflows rejected"],+ ),("workflows_key_already_used_sends_nothing",True,@@ -591,7 +599,7 @@ class TestEmail(BaseTest):use_http=True,)message.add_recipient("[email protected]", "Test", distinct_id="user-1")- if sent_by is None:+ if "transactional email failed" in expected_events:# Workflows may have queued the email, so the error reaches Celery, which retries with the same key.with self.assertRaises(WorkflowsEmailError):message.send(send_async=False, retry=False)@@ -717,6 +725,45 @@ class TestEmail(BaseTest):]self.assertEqual(providers, expected_providers)+ @patch("posthog.email.ph_scoped_capture")+ @patch("posthog.email.posthoganalytics.feature_enabled")+ @patch("posthog.email.requests.post")+ def test_batch_retry_keeps_an_unreachable_recipient_off_customer_io(+ self, mock_post: MagicMock, mock_feature_enabled: MagicMock, _mock_capture: MagicMock+ ) -> None:+ mock_feature_enabled.return_value = True+ unreachable = MagicMock(status_code=422, text="error")+ unreachable.json.return_value = {"type": "invalid_request", "code": "recipient_unreachable"}+ unavailable = MagicMock(status_code=503, text="error")+ unavailable.json.return_value = {"code": "delivery_unavailable"}+ accepted = MagicMock(status_code=202, json=MagicMock(return_value={"message_id": "m1"}))+ mock_post.side_effect = [unreachable, unavailable, unreachable, accepted]++ with (+ override_instance_config("EMAIL_HOST", "localhost"),+ self.settings(+ CUSTOMER_IO_API_KEY="cio-key",+ POSTHOG_TRANSACTIONAL_EMAIL_API_KEY="phs_test",+ POSTHOG_HOST="https://us.posthog.com",+ ),+ patch.dict("posthog.email.WORKFLOWS_TEMPLATE_ID_MAP", {"password_reset": "tmpl-1"}),+ ):+ message = EmailMessage(+ campaign_key="unreachable_batch", subject="Test subject", template_name="password_reset", use_http=True+ )+ message.add_recipient("[email protected]")+ message.add_recipient("[email protected]")+ with self.assertRaises(WorkflowsEmailError):+ message.send(send_async=False, retry=False)+ mock_feature_enabled.return_value = False+ message.send(send_async=False, retry=False)++ self.assertTrue(all(call.args[0].startswith("https://us.posthog.com") for call in mock_post.call_args_list))+ self.assertEqual(+ [call.kwargs["json"]["to"] for call in mock_post.call_args_list],+ )+@patch("posthog.email.ph_scoped_capture")@patch("posthog.email.posthoganalytics.feature_enabled")@patch("posthog.email.requests.post")