Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(email): retry transactional emails after a lost database connection (#123938)

Harley Alexander0dc392a848e8
diff --git a/posthog/email.py b/posthog/email.py
index 4a1f0df47af7..b93a8b5c4e5b 100644
--- a/posthog/email.py
+++ b/posthog/email.py
@@ -15,7 +15,7 @@ from django.conf import settings
from django.core import exceptions, mail
from django.core.cache import cache
from django.core.mail.backends.smtp import EmailBackend
-from django.db import transaction
+from django.db import InterfaceError, OperationalError, transaction
from django.template.loader import get_template
from django.utils import timezone
from django.utils.module_loading import import_string
@@ -25,7 +25,7 @@ import requests
import structlog
import css_inline
import posthoganalytics
-from celery import shared_task
+from celery import current_task, shared_task
from lxml import html as lxml_html
from prometheus_client import Counter
@@ -108,7 +108,9 @@ EMAIL_TASK_KWARGS = {
"ignore_result": True,
"autoretry_for": (Exception,),
"max_retries": 3,
- "retry_backoff": True,
+ # Retries wait 5s, 10s and 20s. Without jitter the last attempt cannot come within a second of the first.
+ "retry_backoff": 5,
+ "retry_jitter": False,
}
# Failure rate as an alertable time series. Labelled only by outcome+transport to bound
@@ -281,6 +283,12 @@ class WorkflowsEmailError(Exception):
self.status_code = status_code
+# Errors that a later attempt can fix. A database error can come before the send, when the record is read, or after
+# it, when the record is marked sent. A Workflows send is pinned before that write, so its retry replays under the
+# same idempotency key. A Customer.io send can go out a second time.
+RETRYABLE_HTTP_SEND_ERRORS = (WorkflowsEmailError, OperationalError, InterfaceError)
+
+
def _workflows_error_allows_fallback(err: WorkflowsEmailError) -> bool:
"""Whether Customer.io can send the email without the risk of a duplicate."""
if err.status_code is None:
@@ -506,7 +514,10 @@ def _send_one(
workflows_template_id = get_workflows_template_id(template_name, dest, pinned_template_id)
if workflows_template_id:
try:
- return "workflows", _send_one_via_workflows(dest, campaign_key, workflows_template_id, properties)
+ response = _send_one_via_workflows(dest, campaign_key, workflows_template_id, properties)
+ # Marking the record sent can still fail, and the retry must then replay this send, not move providers.
+ _pin_to_workflows(idempotency_key, workflows_template_id)
+ return "workflows", response
except WorkflowsEmailError as err:
if pinned_template_id or not _workflows_error_allows_fallback(err):
_pin_to_workflows(idempotency_key, workflows_template_id)
@@ -529,6 +540,7 @@ def _send_via_http(
campaign_key: str,
template_name: str,
properties: dict,
+ will_retry: bool = True,
) -> None:
"""Sends emails using the Customer.io API, or the Workflows transactional email API where the flag matches"""
if not is_http_email_service_available():
@@ -536,11 +548,16 @@ def _send_via_http(
# A dedicated client, because the global one may never flush before a Celery worker exits.
with ph_scoped_capture(event_region=get_instance_region()) as capture:
- _send_via_http_batch(capture, to, campaign_key, template_name, properties)
+ _send_via_http_batch(capture, to, campaign_key, template_name, properties, will_retry)
def _send_via_http_batch(
- capture: ScopedCapture, to: list[dict[str, str]], campaign_key: str, template_name: str, properties: dict
+ capture: ScopedCapture,
+ to: list[dict[str, str]],
+ campaign_key: str,
+ template_name: str,
+ properties: dict,
+ will_retry: bool = True,
) -> None:
sent_count = 0
already_sent_count = 0 # delivered in a prior run — not a failure if the batch later aborts
@@ -583,7 +600,8 @@ def _send_via_http_batch(
except Exception as err:
capture_exception(err) # already logs the traceback via logger.exception
# The exception carries no template name, so this event is what lets a dashboard count failures per template.
- retrying = isinstance(err, WorkflowsEmailError)
+ retryable = isinstance(err, RETRYABLE_HTTP_SEND_ERRORS)
+ retrying = retryable and will_retry
if dest:
_capture_email_event(
capture,
@@ -600,9 +618,9 @@ def _send_via_http_batch(
# Count every recipient that did not get through (the failing one + any not yet attempted),
# so `failed` shares the per-recipient unit with `sent` instead of registering once per batch.
EMAIL_SEND_COUNTER.labels(outcome="failed", transport="http").inc(len(to) - already_sent_count - sent_count)
- if retrying:
- # Workflows may have queued this email. The Celery retry sends the same idempotency key, so Workflows
- # returns the queued email instead of sending a second one, and recipients already sent are skipped.
+ if retryable:
+ # The retry sends the same idempotency key, so Workflows returns an email it already queued instead of
+ # sending a second one, and recipients already marked sent are skipped.
raise
@@ -826,6 +844,16 @@ def sanitize_email_properties(properties: dict[str, Any] | None) -> dict[str, An
return {k: sanitize_value(v, key=k) for k, v in properties.items()}
+def _send_will_retry(retry_in_worker: bool) -> bool:
+ """Whether a retryable failure of this send gets another attempt."""
+ if retry_in_worker:
+ return True
+ task = current_task
+ if not task or task.name != "posthog.email._send_email" or task.request.is_eager:
+ return False
+ return task.request.retries < task.max_retries
+
+
def _send_email_now(
campaign_key: str,
to: list[dict[str, str]],
@@ -846,8 +874,9 @@ def _send_email_now(
campaign_key=campaign_key,
template_name=template_name,
properties=properties or {},
+ will_retry=_send_will_retry(retry_in_worker),
)
- except WorkflowsEmailError:
+ except RETRYABLE_HTTP_SEND_ERRORS:
if not retry_in_worker:
raise
# The retry sends the same idempotency key, so Workflows does not send the email twice.
diff --git a/posthog/test/test_email.py b/posthog/test/test_email.py
index 72b060a57c99..bdd324af7634 100644
--- a/posthog/test/test_email.py
+++ b/posthog/test/test_email.py
@@ -1,6 +1,7 @@
import hashlib
import smtplib
import datetime
+import contextlib
import dataclasses
from decimal import Decimal
from typing import Any
@@ -15,6 +16,7 @@ from django.conf import settings
from django.core import mail
from django.core.cache import cache
from django.core.exceptions import ImproperlyConfigured
+from django.db import OperationalError
from django.test import override_settings
from django.utils import timezone
@@ -309,6 +311,38 @@ class TestEmail(BaseTest):
after = REGISTRY.get_sample_value("posthog_email_send_total", failed_labels) or 0.0
self.assertEqual(after - before, 2.0) # b failed + c never attempted; a succeeded
+ @parameterized.expand([("attempts_left", True), ("last_attempt", False)])
+ @patch("posthog.email.ph_scoped_capture")
+ @patch("posthog.email.requests.post")
+ def test_send_via_http_reraises_a_lost_database_connection_so_celery_retries(
+ self, _name: str, will_retry: bool, mock_post: MagicMock, mock_scoped_capture: MagicMock
+ ) -> None:
+ mock_capture = mock_scoped_capture.return_value.__enter__.return_value
+ lost_connection = OperationalError("could not translate host name to address")
+
+ with (
+ override_instance_config("EMAIL_HOST", "localhost"),
+ self.settings(CUSTOMER_IO_API_KEY="test-key"),
+ patch("posthog.email.MessagingRecord.objects.get_or_create", side_effect=lost_connection),
+ self.assertRaises(OperationalError),
+ ):
+ _send_via_http(
+ to=[{"raw_email": "[email protected]"}],
+ campaign_key="db_blip",
+ template_name="2fa_enabled",
+ properties={},
+ will_retry=will_retry,
+ )
+
+ mock_post.assert_not_called()
+ failed = [
+ c.kwargs for c in mock_capture.call_args_list if c.kwargs.get("event") == "transactional email failed"
+ ]
+ self.assertEqual(
+ [(f["properties"]["error_type"], f["properties"]["retrying"]) for f in failed],
+ [("OperationalError", will_retry)],
+ )
+
@patch("posthog.email.ph_scoped_capture")
@patch("posthog.email.requests.post")
def test_send_via_http_success(self, mock_post, mock_scoped_capture) -> None:
@@ -683,18 +717,76 @@ class TestEmail(BaseTest):
]
self.assertEqual(providers, expected_providers)
+ @patch("posthog.email.ph_scoped_capture")
+ @patch("posthog.email.posthoganalytics.feature_enabled")
+ @patch("posthog.email.requests.post")
+ def test_retry_after_failing_to_mark_a_workflows_send_replays_it_on_workflows(
+ self, mock_post: MagicMock, mock_feature_enabled: MagicMock, _mock_capture: MagicMock
+ ) -> None:
+ mock_feature_enabled.side_effect = [True, False]
+ accepted = MagicMock(status_code=202, json=MagicMock(return_value={"message_id": "m1"}))
+ mock_post.side_effect = [accepted, accepted]
+ original_save = MessagingRecord.save
+ failed: list[MessagingRecord] = []
+
+ def lose_connection_on_first_sent_mark(record: MessagingRecord, *args: Any, **kwargs: Any) -> None:
+ if record.sent_at and not failed:
+ failed.append(record)
+ raise OperationalError("could not translate host name to address")
+ original_save(record, *args, **kwargs)
+
+ with (
+ override_instance_config("EMAIL_HOST", "localhost"),
+ self.settings(
+ CUSTOMER_IO_API_KEY="cio-key",
+ POSTHOG_TRANSACTIONAL_EMAIL_API_KEY="phs_test",
+ POSTHOG_HOST="https://us.posthog.com",
+ ),
+ patch.dict("posthog.email.WORKFLOWS_TEMPLATE_ID_MAP", {"password_reset": "tmpl-1"}),
+ patch.object(MessagingRecord, "save", autospec=True, side_effect=lose_connection_on_first_sent_mark),
+ ):
+ message = EmailMessage(
+ campaign_key="mark_failed_campaign",
+ subject="Test subject",
+ template_name="password_reset",
+ use_http=True,
+ )
+ message.add_recipient("[email protected]")
+ with self.assertRaises(OperationalError):
+ message.send(send_async=False, retry=False)
+ message.send(send_async=False, retry=False)
+
+ # Workflows already queued the first email, so Customer.io sending the retry would deliver it twice.
+ providers = [
+ "workflows" if call.args[0].startswith("https://us.posthog.com") else "customer_io"
+ for call in mock_post.call_args_list
+ ]
+ self.assertEqual(providers, ["workflows", "workflows"])
+
+ @parameterized.expand(
+ [
+ ("workflows_timeout", None, 1),
+ ("lost_database_connection", OperationalError("could not translate host name to address"), 0),
+ ]
+ )
@patch.object(_send_email, "apply_async")
@patch("posthog.email.ph_scoped_capture")
@patch("posthog.email.posthoganalytics.feature_enabled", return_value=True)
@patch("posthog.email.requests.post", side_effect=requests.ReadTimeout("no response"))
- def test_inline_send_hands_an_uncertain_workflows_send_to_a_worker(
+ def test_inline_send_hands_a_retryable_failure_to_a_worker(
self,
+ _name: str,
+ record_error: Exception | None,
+ expected_posts: int,
mock_post: MagicMock,
_mock_feature_enabled: MagicMock,
_mock_capture: MagicMock,
mock_apply_async: MagicMock,
) -> None:
with (
+ patch("posthog.email.MessagingRecord.objects.get_or_create", side_effect=record_error)
+ if record_error
+ else contextlib.nullcontext(),
override_instance_config("EMAIL_HOST", "localhost"),
self.settings(
CUSTOMER_IO_API_KEY="cio-key",
@@ -709,8 +801,8 @@ class TestEmail(BaseTest):
message.add_recipient("[email protected]")
message.send(send_async=False)
- # Eager retries would post once per retry inside the request, each up to the 10s timeout.
- self.assertEqual(mock_post.call_count, 1)
+ # Eager retries would run at once inside the request, with no backoff for the failure to clear.
+ self.assertEqual(mock_post.call_count, expected_posts)
mock_apply_async.assert_called_once()
retry_kwargs = mock_apply_async.call_args.kwargs["kwargs"]
self.assertEqual(retry_kwargs["campaign_key"], "inline_campaign")