:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(email): retry transactional emails after a lost database connection (#123938)
Harley Alexander0dc392a848e8
diff --git a/posthog/email.py b/posthog/email.pyindex 4a1f0df47af7..b93a8b5c4e5b 100644--- a/posthog/email.py+++ b/posthog/email.py@@ -15,7 +15,7 @@ from django.conf import settingsfrom django.core import exceptions, mailfrom django.core.cache import cachefrom django.core.mail.backends.smtp import EmailBackend-from django.db import transaction+from django.db import InterfaceError, OperationalError, transactionfrom django.template.loader import get_templatefrom django.utils import timezonefrom django.utils.module_loading import import_string@@ -25,7 +25,7 @@ import requestsimport structlogimport css_inlineimport posthoganalytics-from celery import shared_task+from celery import current_task, shared_taskfrom lxml import html as lxml_htmlfrom prometheus_client import Counter@@ -108,7 +108,9 @@ EMAIL_TASK_KWARGS = {"ignore_result": True,"autoretry_for": (Exception,),"max_retries": 3,- "retry_backoff": True,+ # Retries wait 5s, 10s and 20s. Without jitter the last attempt cannot come within a second of the first.+ "retry_backoff": 5,+ "retry_jitter": False,}# Failure rate as an alertable time series. Labelled only by outcome+transport to bound@@ -281,6 +283,12 @@ class WorkflowsEmailError(Exception):self.status_code = status_code+# Errors that a later attempt can fix. A database error can come before the send, when the record is read, or after+# it, when the record is marked sent. A Workflows send is pinned before that write, so its retry replays under the+# same idempotency key. A Customer.io send can go out a second time.+RETRYABLE_HTTP_SEND_ERRORS = (WorkflowsEmailError, OperationalError, InterfaceError)++def _workflows_error_allows_fallback(err: WorkflowsEmailError) -> bool:"""Whether Customer.io can send the email without the risk of a duplicate."""if err.status_code is None:@@ -506,7 +514,10 @@ def _send_one(workflows_template_id = get_workflows_template_id(template_name, dest, pinned_template_id)if workflows_template_id:try:- return "workflows", _send_one_via_workflows(dest, campaign_key, workflows_template_id, properties)+ response = _send_one_via_workflows(dest, campaign_key, workflows_template_id, properties)+ # Marking the record sent can still fail, and the retry must then replay this send, not move providers.+ _pin_to_workflows(idempotency_key, workflows_template_id)+ return "workflows", responseexcept WorkflowsEmailError as err:if pinned_template_id or not _workflows_error_allows_fallback(err):_pin_to_workflows(idempotency_key, workflows_template_id)@@ -529,6 +540,7 @@ def _send_via_http(campaign_key: str,template_name: str,properties: dict,+ will_retry: bool = True,) -> None:"""Sends emails using the Customer.io API, or the Workflows transactional email API where the flag matches"""if not is_http_email_service_available():@@ -536,11 +548,16 @@ def _send_via_http(# A dedicated client, because the global one may never flush before a Celery worker exits.with ph_scoped_capture(event_region=get_instance_region()) as capture:- _send_via_http_batch(capture, to, campaign_key, template_name, properties)+ _send_via_http_batch(capture, to, campaign_key, template_name, properties, will_retry)def _send_via_http_batch(- capture: ScopedCapture, to: list[dict[str, str]], campaign_key: str, template_name: str, properties: dict+ capture: ScopedCapture,+ to: list[dict[str, str]],+ campaign_key: str,+ template_name: str,+ properties: dict,+ will_retry: bool = True,) -> None:sent_count = 0already_sent_count = 0 # delivered in a prior run — not a failure if the batch later aborts@@ -583,7 +600,8 @@ def _send_via_http_batch(except Exception as err:capture_exception(err) # already logs the traceback via logger.exception# The exception carries no template name, so this event is what lets a dashboard count failures per template.- retrying = isinstance(err, WorkflowsEmailError)+ retryable = isinstance(err, RETRYABLE_HTTP_SEND_ERRORS)+ retrying = retryable and will_retryif dest:_capture_email_event(capture,@@ -600,9 +618,9 @@ def _send_via_http_batch(# Count every recipient that did not get through (the failing one + any not yet attempted),# so `failed` shares the per-recipient unit with `sent` instead of registering once per batch.EMAIL_SEND_COUNTER.labels(outcome="failed", transport="http").inc(len(to) - already_sent_count - sent_count)- if retrying:- # Workflows may have queued this email. The Celery retry sends the same idempotency key, so Workflows- # returns the queued email instead of sending a second one, and recipients already sent are skipped.+ if retryable:+ # The retry sends the same idempotency key, so Workflows returns an email it already queued instead of+ # sending a second one, and recipients already marked sent are skipped.raise@@ -826,6 +844,16 @@ def sanitize_email_properties(properties: dict[str, Any] | None) -> dict[str, Anreturn {k: sanitize_value(v, key=k) for k, v in properties.items()}+def _send_will_retry(retry_in_worker: bool) -> bool:+ """Whether a retryable failure of this send gets another attempt."""+ if retry_in_worker:+ return True+ task = current_task+ if not task or task.name != "posthog.email._send_email" or task.request.is_eager:+ return False+ return task.request.retries < task.max_retries++def _send_email_now(campaign_key: str,to: list[dict[str, str]],@@ -846,8 +874,9 @@ def _send_email_now(campaign_key=campaign_key,template_name=template_name,properties=properties or {},+ will_retry=_send_will_retry(retry_in_worker),)- except WorkflowsEmailError:+ except RETRYABLE_HTTP_SEND_ERRORS:if not retry_in_worker:raise# The retry sends the same idempotency key, so Workflows does not send the email twice.diff --git a/posthog/test/test_email.py b/posthog/test/test_email.pyindex 72b060a57c99..bdd324af7634 100644--- a/posthog/test/test_email.py+++ b/posthog/test/test_email.py@@ -1,6 +1,7 @@import hashlibimport smtplibimport datetime+import contextlibimport dataclassesfrom decimal import Decimalfrom typing import Any@@ -15,6 +16,7 @@ from django.conf import settingsfrom django.core import mailfrom django.core.cache import cachefrom django.core.exceptions import ImproperlyConfigured+from django.db import OperationalErrorfrom django.test import override_settingsfrom django.utils import timezone@@ -309,6 +311,38 @@ class TestEmail(BaseTest):after = REGISTRY.get_sample_value("posthog_email_send_total", failed_labels) or 0.0self.assertEqual(after - before, 2.0) # b failed + c never attempted; a succeeded+ @parameterized.expand([("attempts_left", True), ("last_attempt", False)])+ @patch("posthog.email.ph_scoped_capture")+ @patch("posthog.email.requests.post")+ def test_send_via_http_reraises_a_lost_database_connection_so_celery_retries(+ self, _name: str, will_retry: bool, mock_post: MagicMock, mock_scoped_capture: MagicMock+ ) -> None:+ mock_capture = mock_scoped_capture.return_value.__enter__.return_value+ lost_connection = OperationalError("could not translate host name to address")++ with (+ override_instance_config("EMAIL_HOST", "localhost"),+ self.settings(CUSTOMER_IO_API_KEY="test-key"),+ patch("posthog.email.MessagingRecord.objects.get_or_create", side_effect=lost_connection),+ self.assertRaises(OperationalError),+ ):+ _send_via_http(+ to=[{"raw_email": "[email protected]"}],+ campaign_key="db_blip",+ template_name="2fa_enabled",+ properties={},+ will_retry=will_retry,+ )++ mock_post.assert_not_called()+ failed = [+ c.kwargs for c in mock_capture.call_args_list if c.kwargs.get("event") == "transactional email failed"+ ]+ self.assertEqual(+ [(f["properties"]["error_type"], f["properties"]["retrying"]) for f in failed],+ [("OperationalError", will_retry)],+ )+@patch("posthog.email.ph_scoped_capture")@patch("posthog.email.requests.post")def test_send_via_http_success(self, mock_post, mock_scoped_capture) -> None:@@ -683,18 +717,76 @@ class TestEmail(BaseTest):]self.assertEqual(providers, expected_providers)+ @patch("posthog.email.ph_scoped_capture")+ @patch("posthog.email.posthoganalytics.feature_enabled")+ @patch("posthog.email.requests.post")+ def test_retry_after_failing_to_mark_a_workflows_send_replays_it_on_workflows(+ self, mock_post: MagicMock, mock_feature_enabled: MagicMock, _mock_capture: MagicMock+ ) -> None:+ mock_feature_enabled.side_effect = [True, False]+ accepted = MagicMock(status_code=202, json=MagicMock(return_value={"message_id": "m1"}))+ mock_post.side_effect = [accepted, accepted]+ original_save = MessagingRecord.save+ failed: list[MessagingRecord] = []++ def lose_connection_on_first_sent_mark(record: MessagingRecord, *args: Any, **kwargs: Any) -> None:+ if record.sent_at and not failed:+ failed.append(record)+ raise OperationalError("could not translate host name to address")+ original_save(record, *args, **kwargs)++ with (+ override_instance_config("EMAIL_HOST", "localhost"),+ self.settings(+ CUSTOMER_IO_API_KEY="cio-key",+ POSTHOG_TRANSACTIONAL_EMAIL_API_KEY="phs_test",+ POSTHOG_HOST="https://us.posthog.com",+ ),+ patch.dict("posthog.email.WORKFLOWS_TEMPLATE_ID_MAP", {"password_reset": "tmpl-1"}),+ patch.object(MessagingRecord, "save", autospec=True, side_effect=lose_connection_on_first_sent_mark),+ ):+ message = EmailMessage(+ campaign_key="mark_failed_campaign",+ subject="Test subject",+ template_name="password_reset",+ use_http=True,+ )+ message.add_recipient("[email protected]")+ with self.assertRaises(OperationalError):+ message.send(send_async=False, retry=False)+ message.send(send_async=False, retry=False)++ # Workflows already queued the first email, so Customer.io sending the retry would deliver it twice.+ providers = [+ "workflows" if call.args[0].startswith("https://us.posthog.com") else "customer_io"+ for call in mock_post.call_args_list+ ]+ self.assertEqual(providers, ["workflows", "workflows"])++ @parameterized.expand(+ [+ ("workflows_timeout", None, 1),+ ("lost_database_connection", OperationalError("could not translate host name to address"), 0),+ ]+ )@patch.object(_send_email, "apply_async")@patch("posthog.email.ph_scoped_capture")@patch("posthog.email.posthoganalytics.feature_enabled", return_value=True)@patch("posthog.email.requests.post", side_effect=requests.ReadTimeout("no response"))- def test_inline_send_hands_an_uncertain_workflows_send_to_a_worker(+ def test_inline_send_hands_a_retryable_failure_to_a_worker(self,+ _name: str,+ record_error: Exception | None,+ expected_posts: int,mock_post: MagicMock,_mock_feature_enabled: MagicMock,_mock_capture: MagicMock,mock_apply_async: MagicMock,) -> None:with (+ patch("posthog.email.MessagingRecord.objects.get_or_create", side_effect=record_error)+ if record_error+ else contextlib.nullcontext(),override_instance_config("EMAIL_HOST", "localhost"),self.settings(CUSTOMER_IO_API_KEY="cio-key",@@ -709,8 +801,8 @@ class TestEmail(BaseTest):message.add_recipient("[email protected]")message.send(send_async=False)- # Eager retries would post once per retry inside the request, each up to the 10s timeout.- self.assertEqual(mock_post.call_count, 1)+ # Eager retries would run at once inside the request, with no backoff for the failure to clear.+ self.assertEqual(mock_post.call_count, expected_posts)mock_apply_async.assert_called_once()retry_kwargs = mock_apply_async.call_args.kwargs["kwargs"]self.assertEqual(retry_kwargs["campaign_key"], "inline_campaign")