:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(alerts): email project members for warehouse sync alerts (#123922)
Tom Owers0d939f7fcc1a
diff --git a/posthog/tasks/email.py b/posthog/tasks/email.pyindex 3b37f96dc0c1..7e74281542cb 100644--- a/posthog/tasks/email.py+++ b/posthog/tasks/email.py@@ -1269,7 +1269,7 @@ def send_external_data_failure_digest(team_id: int, schemas: list[dict[str, Any]return delivered-def _filter_members_by_external_data_source_access(+def filter_members_by_external_data_source_access(memberships: list[OrganizationMembership], team: Team) -> list[OrganizationMembership]:"""Drop members who cannot view destinations, which `ExternalDataDestinationViewSet` gates on `external_data_source`.@@ -1312,7 +1312,7 @@ def send_warehouse_destination_paused(memberships_to_email = get_members_to_notify_for_pipeline_error(team, failure_rate=1.0, pipeline_id=f"warehouse_destination:{destination_id}")- memberships_to_email = _filter_members_by_external_data_source_access(memberships_to_email, team)+ memberships_to_email = filter_members_by_external_data_source_access(memberships_to_email, team)if not memberships_to_email:returndiff --git a/products/alerts/backend/facade/contracts.py b/products/alerts/backend/facade/contracts.pyindex 0fadf08ad53d..8ec9d2e436aa 100644--- a/products/alerts/backend/facade/contracts.py+++ b/products/alerts/backend/facade/contracts.py@@ -2,14 +2,15 @@Framework-free values this product hands to its consumers. The alerting vocabulary every sourceshares lives in `products.alerts_platform.backend.facade.contracts`; what stays here is what only-an insight alert means.+an insight alert means, and the hooks other sources use to reach this product's destinations."""from __future__ import annotations+from collections.abc import Collectionfrom datetime import datetimefrom enum import StrEnum-from typing import Any, Final+from typing import Any, Final, Protocolfrom uuid import UUIDfrom posthog.dataclasses import frozen@@ -139,6 +140,15 @@ class AlertInternalEventsNotDelivered(Exception):"""The broker did not take every event, so the caller retries."""+class AlertEmailRecipientResolver(Protocol):+ """The email addresses one alert reaches besides its destinations, for a source that registers one.++ The source applies its own access rules, so every address it returns may read what the alert says.+ """++ def __call__(self, *, team_id: int, alert_id: str) -> Collection[str]: ...++@frozenclass MovedAlertRuntime:"""Where an alert the platform owns stands, in the insight alert's terms."""diff --git a/products/alerts/backend/facade/destinations.py b/products/alerts/backend/facade/destinations.pyindex 1913465b3e9d..18000086eda0 100644--- a/products/alerts/backend/facade/destinations.py+++ b/products/alerts/backend/facade/destinations.py@@ -29,6 +29,7 @@ from products.alerts_platform.backend.facade.contracts import ()from ..logic import (+ alert_email,delivery_destinations,delivery_path,destination_bucket_report,@@ -38,6 +39,7 @@ from ..logic import (insight_alert_destinations,)from .contracts import (+ AlertEmailRecipientResolver,AlertInternalEvent,DeliveryPathChoice,DestinationBucket,@@ -258,6 +260,16 @@ def list_delivery_destination_groups()+def register_email_recipient_resolver(*, event_ids: Collection[str], resolver: AlertEmailRecipientResolver) -> None:+ """Platform delivery of these events also emails the addresses `resolver` returns for the alert.++ Register the events a source maps firing and resolved onto, and not its incident events, or an+ incident would send a second email. Call it from the source's `ready()`. Raises when another+ resolver already owns one of the events.+ """+ alert_email.register_recipient_resolver(event_ids=event_ids, resolver=resolver)++def classify_alert_destinations(*, team_id: int, alert_id: str, allowed_event_ids: Collection[str]) -> tuple[DestinationEligibility, ...]:diff --git a/products/alerts/backend/logic/alert_email.py b/products/alerts/backend/logic/alert_email.pyindex f63c26a7d576..87cb7012676c 100644--- a/products/alerts/backend/logic/alert_email.py+++ b/products/alerts/backend/logic/alert_email.py@@ -1,16 +1,21 @@-"""Who an insight alert emails, for the production check and for the shared platform.+"""Who an alert emails besides its HogFunction destinations.-An insight alert emails its subscribed users besides any HogFunction destinations. Both senders-read recipients here, so a user who loses access to the insight stops receiving either.+An insight alert emails its subscribed users. Both senders, the production check and the shared+platform, read recipients here, so a user who loses access to the insight stops receiving either.++Another source registers a resolver for the event ids it maps firing and resolved onto, and the+platform then emails the addresses that resolver returns for each alert. The source owns the+access rules, because only it knows who may read what its alerts say."""from __future__ import annotations-from collections.abc import Collection+from collections.abc import Collection, Iterablefrom typing import Finalfrom uuid import UUIDfrom products.access_control.backend.facade.user_access_control import UserAccessControl+from products.alerts.backend.facade.contracts import AlertEmailRecipientResolverfrom products.alerts.backend.models.alert import AlertConfigurationfrom products.alerts_platform.backend.facade.contracts import (AlertDestinationData,@@ -55,8 +60,43 @@ def recipients_with_access(alert: AlertConfiguration) -> list[tuple[int, str]]:def insight_email_groups(recipients: Collection[tuple[int, str]]) -> list[AlertDestinationGroup]:- addresses = sorted({email for _, email in recipients if email})- if not addresses:+ return email_groups(email for _, email in recipients)+++def email_groups(addresses: Iterable[str]) -> list[AlertDestinationGroup]:+ # Sorted and deduplicated, because delivery tells destinations apart by their data. The same+ # people in another order would read as a second destination.+ unique = sorted({address for address in addresses if address})+ if not unique:return []- data: AlertDestinationData = {"type": DestinationType.EMAIL, "email_addresses": addresses}+ data: AlertDestinationData = {"type": DestinationType.EMAIL, "email_addresses": unique}return [AlertDestinationGroup(hog_function_ids=(), data=data, fully_enabled=True)]+++_recipient_resolvers: dict[str, AlertEmailRecipientResolver] = {}+++def register_recipient_resolver(*, event_ids: Collection[str], resolver: AlertEmailRecipientResolver) -> None:+ for event_id in event_ids:+ if event_id in SUBSCRIBER_EVENT_IDS:+ raise RuntimeError(f"{event_id} already emails the subscribers of an insight alert.")+ registered = _recipient_resolvers.get(event_id)+ # One owner per event. A second registrant would silently replace who the first one emails.+ if registered is not None and registered is not resolver:+ raise RuntimeError(f"A different email recipient resolver is already registered for {event_id}.")+ for event_id in event_ids:+ _recipient_resolvers[event_id] = resolver+++def registered_email_groups(+ *, team_id: int, alert_id: str, allowed_event_ids: Collection[str]+) -> list[AlertDestinationGroup]:+ """One email group for the addresses that the resolvers of these events return for the alert."""+ resolvers = {+ id(resolver): resolver+ for event_id in allowed_event_ids+ if (resolver := _recipient_resolvers.get(event_id)) is not None+ }+ return email_groups(+ address for resolver in resolvers.values() for address in resolver(team_id=team_id, alert_id=alert_id)+ )diff --git a/products/alerts/backend/logic/delivery_destinations.py b/products/alerts/backend/logic/delivery_destinations.pyindex 7ef4dfb6a7c6..98cd8ef35efa 100644--- a/products/alerts/backend/logic/delivery_destinations.py+++ b/products/alerts/backend/logic/delivery_destinations.py@@ -9,6 +9,7 @@ from products.alerts.backend.logic.alert_email import (insight_alert,insight_email_groups,recipients_with_access,+ registered_email_groups,)from products.alerts.backend.logic.alert_in_app import insight_in_app_groupsfrom products.alerts.backend.logic.destinations import list_alert_destination_groups@@ -19,12 +20,15 @@ def list_delivery_destination_groups(*, team_id: int, alert_id: str, allowed_event_ids: Collection[str]) -> list[AlertDestinationGroup]:"""An alert's HogFunction destinations, plus its subscribers' email and in-app notifications- for an insight alert.+ for an insight alert, plus the email recipients another source registered for its events.Separate from `list_alert_destination_groups`, which the alert APIs list and delete through. Asubscriber group there would show up as a destination a person could delete."""- groups = list_alert_destination_groups(team_id=team_id, alert_id=alert_id, allowed_event_ids=allowed_event_ids)+ groups = [+ *list_alert_destination_groups(team_id=team_id, alert_id=alert_id, allowed_event_ids=allowed_event_ids),+ *registered_email_groups(team_id=team_id, alert_id=alert_id, allowed_event_ids=allowed_event_ids),+ ]event_ids = SUBSCRIBER_EVENT_IDS.intersection(allowed_event_ids)if not event_ids:return groupsdiff --git a/products/alerts/backend/test/test_alert_email.py b/products/alerts/backend/test/test_alert_email.pyindex 06b36f8ebf0c..ea1d4b563665 100644--- a/products/alerts/backend/test/test_alert_email.py+++ b/products/alerts/backend/test/test_alert_email.py@@ -1,12 +1,20 @@+import pytestfrom posthog.test.base import APIBaseTestfrom unittest.mock import patch+from django.test import SimpleTestCase+from parameterized import parameterizedfrom posthog.models.user import Userfrom products.access_control.backend.facade.user_access_control import UserAccessControl-from products.alerts.backend.facade.destinations import list_alert_destination_groups, list_delivery_destination_groups+from products.alerts.backend.facade.destinations import (+ list_alert_destination_groups,+ list_delivery_destination_groups,+ register_email_recipient_resolver,+)+from products.alerts.backend.logic import alert_emailfrom products.alerts.backend.logic.alert_email import INSIGHT_ALERT_ERRORED_EVENT_IDfrom products.alerts.backend.models.alert import AlertConfiguration, AlertSubscriptionfrom products.alerts.backend.test.insight_alerts import create_insight_alert@@ -79,3 +87,30 @@ class TestInsightAlertEmailDestination(APIBaseTest):)assert groups == []+++class TestEmailRecipientRegistration(SimpleTestCase):+ @staticmethod+ def _nobody(*, team_id: int, alert_id: str) -> list[str]:+ return []++ @staticmethod+ def _somebody(*, team_id: int, alert_id: str) -> list[str]:+ return ["[email protected]"]++ @parameterized.expand(+ [+ ("an_event_another_source_registered", "$registered_firing", "already registered"),+ ("an_insight_subscriber_event", LEGACY_INSIGHT_ALERT_EVENT, "subscribers of an insight alert"),+ ]+ )+ def test_a_second_owner_for_an_event_is_refused(self, _name: str, event_id: str, message: str) -> None:+ with patch.dict(alert_email._recipient_resolvers, clear=True):+ register_email_recipient_resolver(event_ids=["$registered_firing"], resolver=self._nobody)+ # Registering the same resolver again, as a second `ready()` does, is not a conflict.+ register_email_recipient_resolver(event_ids=["$registered_firing"], resolver=self._nobody)++ with pytest.raises(RuntimeError, match=message):+ register_email_recipient_resolver(event_ids=["$unclaimed", event_id], resolver=self._somebody)++ assert alert_email._recipient_resolvers == {"$registered_firing": self._nobody}diff --git a/products/alerts_platform/backend/facade/testing.py b/products/alerts_platform/backend/facade/testing.pyindex 6d3f4a2e42c3..2677da4fab6e 100644--- a/products/alerts_platform/backend/facade/testing.py+++ b/products/alerts_platform/backend/facade/testing.py@@ -13,7 +13,9 @@ from datetime import datetimefrom typing import Anyfrom uuid import UUID+from products.alerts_platform.backend.delivery.evaluation import deliver_evaluationfrom products.alerts_platform.backend.facade.contracts import (+ AlertDeliveryRequest,DeliveryPath,PlatformAlertSnapshot,PlatformConfigurationSnapshot,@@ -70,6 +72,11 @@ def set_due_at(configuration_id: UUID, due_at: datetime) -> None:PlatformAlertConfiguration.objects.filter(id=configuration_id).update(next_check_at=due_at)+def deliver(request: AlertDeliveryRequest) -> int:+ """Send a recorded evaluation's delivery as the delivery activity does. Returns how many destinations took it."""+ return deliver_evaluation(request).sent++def count_still_due(configuration_ids: Iterable[UUID], *, at: datetime) -> int:"""How many of these configurations a tick would still pick up at `at`."""# nosemgrep: idor-lookup-without-team — fail-closed model; the caller holds the team scopediff --git a/products/warehouse_sources/backend/sync_alerts/email_recipients.py b/products/warehouse_sources/backend/sync_alerts/email_recipients.pynew file mode 100644index 000000000000..903a5f7e0fb9--- /dev/null+++ b/products/warehouse_sources/backend/sync_alerts/email_recipients.py@@ -0,0 +1,46 @@+"""Who a warehouse alert emails when its `email_project_members` setting is on.++The people the data pipeline error emails reach: active members of the project's organization who+can access the project and did not turn those emails off. One rule for both, so a member who opts+out of pipeline error emails also stops receiving these. Members who cannot view warehouse sources+are left out, as for the destination paused email, because these emails name sources and their errors.+"""++from typing import Final+from uuid import UUID++from posthog.models import Team+from posthog.tasks.email import filter_members_by_external_data_source_access, get_members_to_notify_for_pipeline_error++from products.alerts_platform.backend.facade.api import get_configuration+from products.alerts_platform.backend.facade.contracts import SourceKind++WAREHOUSE_ALERT_KINDS: Final = (SourceKind.WAREHOUSE_SOURCE.value, SourceKind.WAREHOUSE_DESTINATION.value)+++def project_member_emails(*, team_id: int, alert_id: str) -> list[str]:+ """The addresses a warehouse alert emails, or none when the alert does not email project members.++ `alert_id` is the alert's configuration id, which is the id a warehouse alert's destinations match on.+ """+ try:+ configuration_id = UUID(alert_id)+ except ValueError:+ return []+ configuration = get_configuration(+ team_id=team_id, source_kinds=WAREHOUSE_ALERT_KINDS, configuration_id=configuration_id+ )+ if configuration is None or not configuration.source_config.get("email_project_members"):+ return []+ team = Team.objects.filter(id=team_id).first()+ if team is None:+ return []+ return sorted(+ {+ membership.user.email+ for membership in filter_members_by_external_data_source_access(+ get_members_to_notify_for_pipeline_error(team), team+ )+ if membership.user.email+ }+ )diff --git a/products/warehouse_sources/backend/sync_alerts/registration.py b/products/warehouse_sources/backend/sync_alerts/registration.pyindex 2965670425aa..06573cc10599 100644--- a/products/warehouse_sources/backend/sync_alerts/registration.py+++ b/products/warehouse_sources/backend/sync_alerts/registration.py@@ -1,8 +1,10 @@"""Plugs warehouse alerts into the alerts platform, which imports no source. Called from `ready()`."""+from products.alerts.backend.facade.destinations import register_email_recipient_resolverfrom products.alerts_platform.backend.facade.contracts import SourceConfigSpec, SourceKindfrom products.alerts_platform.backend.facade.delivery import register_source_describerfrom products.alerts_platform.backend.facade.source_configs import register_source_config+from products.warehouse_sources.backend.sync_alerts.destinations import EVENT_IDS_BY_KINDfrom products.warehouse_sources.backend.sync_alerts.wording import (describe_destination_transition,describe_source_transition,@@ -25,9 +27,23 @@ def load_destination_alert_config() -> SourceConfigSpec:return WAREHOUSE_DESTINATION_ALERT_CONFIG+def project_member_emails(*, team_id: int, alert_id: str) -> list[str]:+ from products.warehouse_sources.backend.sync_alerts.email_recipients import ( # noqa: PLC0415 - keeps the platform API and the email tasks off the startup path+ project_member_emails as resolve,+ )++ return resolve(team_id=team_id, alert_id=alert_id)++def register() -> None:# Native delivery only, so no legacy event emitter and no legacy webhook body.register_source_describer(SourceKind.WAREHOUSE_SOURCE, describe_source_transition)register_source_describer(SourceKind.WAREHOUSE_DESTINATION, describe_destination_transition)register_source_config(SourceKind.WAREHOUSE_SOURCE, load_source_alert_config)register_source_config(SourceKind.WAREHOUSE_DESTINATION, load_destination_alert_config)+ # Firing and resolved only. The incident events reach an incident manager, and an email for+ # them would send each person a second copy of the same transition.+ register_email_recipient_resolver(+ event_ids=[event_id for by_kind in EVENT_IDS_BY_KIND.values() for event_id in by_kind.values()],+ resolver=project_member_emails,+ )diff --git a/products/warehouse_sources/backend/tests/test_sync_alerts.py b/products/warehouse_sources/backend/tests/test_sync_alerts.pyindex d2a6cfb42e0e..6ae215eacdc6 100644--- a/products/warehouse_sources/backend/tests/test_sync_alerts.py+++ b/products/warehouse_sources/backend/tests/test_sync_alerts.py@@ -10,10 +10,18 @@ from django.utils import timezonefrom parameterized import parameterizedfrom rest_framework import status+from posthog.constants import AvailableFeature+from posthog.models.organization import Organization, OrganizationMembershipfrom posthog.models.scoping import team_scopefrom posthog.models.team.team import Team+from posthog.models.user import User-from products.alerts.backend.facade.destinations import MAX_DESTINATIONS_PER_ALERT, list_alert_destination_groups+from products.access_control.backend.models.access_control import AccessControl+from products.alerts.backend.facade.destinations import (+ MAX_DESTINATIONS_PER_ALERT,+ list_alert_destination_groups,+ list_delivery_destination_groups,+)from products.alerts_platform.backend.facade import testing as platform_testingfrom products.alerts_platform.backend.facade.api import (create_configuration,@@ -25,6 +33,7 @@ from products.alerts_platform.backend.facade.api import (from products.alerts_platform.backend.facade.contracts import (AlertEventKind,AnnouncedTransition,+ DestinationType,Grouping,GroupingMode,IncidentAction,@@ -40,7 +49,7 @@ from products.warehouse_sources.backend.models.external_data_destination importfrom products.warehouse_sources.backend.models.external_data_schema import ExternalDataSchemafrom products.warehouse_sources.backend.models.external_data_source import ExternalDataSourcefrom products.warehouse_sources.backend.presentation.views import warehouse_alert_destinations as destinations_view-from products.warehouse_sources.backend.sync_alerts.destinations import EVENT_IDS_BY_KIND+from products.warehouse_sources.backend.sync_alerts.destinations import EVENT_IDS, EVENT_IDS_BY_KINDfrom products.warehouse_sources.backend.sync_alerts.evaluation import evaluate_warehouse_alert_batchfrom products.warehouse_sources.backend.sync_alerts.wording import (describe_destination_transition,@@ -615,3 +624,91 @@ class TestWarehouseAlerts(APIBaseTest):assert [(detail.label, detail.value) for detail in description.details] == detailsassert description.data_link is not Noneassert description.data_link.url.endswith(path)++ def _person(self, who: str) -> User:+ if who == "another_organization":+ other = Organization.objects.create(name="Elsewhere")+ return User.objects.create_and_join(other, "[email protected]", None)+ person = User.objects.create_and_join(self.organization, f"{who}@example.com", None)+ if who == "opted_out":+ person.partial_notification_settings = {"plugin_disabled": False}+ person.save()+ if who in ("no_project_access", "no_warehouse_access"):+ self.organization.available_product_features = [+ {"key": AvailableFeature.ACCESS_CONTROL, "name": AvailableFeature.ACCESS_CONTROL}+ ]+ self.organization.save(update_fields=["available_product_features"])+ self.organization_membership.level = OrganizationMembership.Level.ADMIN+ self.organization_membership.save(update_fields=["level"])+ if who == "no_project_access":+ # A private project, which an organization member reaches only through an explicit grant.+ AccessControl.objects.create(+ team=self.team, resource="project", resource_id=str(self.team.id), access_level="none"+ )+ if who == "no_warehouse_access":+ AccessControl.objects.create(team=self.team, resource="external_data_source", access_level="none")+ return person++ @parameterized.expand(+ [+ ("a_member", SourceKind.WAREHOUSE_SOURCE, "member", True, True),+ ("a_member_of_a_destination_alert", SourceKind.WAREHOUSE_DESTINATION, "member", True, True),+ ("nobody_when_the_alert_does_not_email", SourceKind.WAREHOUSE_SOURCE, "member", False, False),+ ("not_a_member_who_turned_off_pipeline_emails", SourceKind.WAREHOUSE_SOURCE, "opted_out", True, False),+ ("not_a_member_without_project_access", SourceKind.WAREHOUSE_SOURCE, "no_project_access", True, False),+ ("not_a_member_without_warehouse_access", SourceKind.WAREHOUSE_SOURCE, "no_warehouse_access", True, False),+ (+ "not_a_member_of_a_destination_alert_without_warehouse_access",+ SourceKind.WAREHOUSE_DESTINATION,+ "no_warehouse_access",+ True,+ False,+ ),+ ("not_a_user_of_another_organization", SourceKind.WAREHOUSE_SOURCE, "another_organization", True, False),+ ]+ )+ def test_an_alert_that_emails_project_members_reaches_who_pipeline_error_emails_reach(+ self, _name: str, kind: SourceKind, who: str, email_project_members: bool, reached: bool+ ) -> None:+ person = self._person(who)+ grouping_key = "source_id" if kind == SourceKind.WAREHOUSE_SOURCE else "destination_id"+ configuration_id = self._alert(kind, {"email_project_members": email_project_members}, grouping_key)+ expected = sorted([self.user.email, person.email] if reached else [self.user.email])++ groups_by_event = {+ event_id: list_delivery_destination_groups(+ team_id=self.team.id, alert_id=str(configuration_id), allowed_event_ids=[event_id]+ )+ for event_id in EVENT_IDS[kind]+ }++ message_event_ids = set(EVENT_IDS_BY_KIND[kind].values())+ assert {event_id: [group.data for group in groups] for event_id, groups in groups_by_event.items()} == {+ event_id: [{"type": DestinationType.EMAIL, "email_addresses": expected}]+ if email_project_members and event_id in message_event_ids+ else []+ for event_id in EVENT_IDS[kind]+ }++ def test_a_failing_table_emails_the_project_members_of_an_alert_that_emails_them(self) -> None:+ self._person("member")+ self._schema(self._source(), "charges", Status.FAILED)+ configuration_id = self._alert(+ SourceKind.WAREHOUSE_SOURCE, {"group_by": "source", "email_project_members": True}, "source_id"+ )+ evaluation = self._evaluate(SourceKind.WAREHOUSE_SOURCE)+ # The history that delivery reads is written when the record commits.+ with team_scope(self.team.id), self.captureOnCommitCallbacks(execute=True):+ record_outcomes(self.team.id, list(evaluation.outcomes), self.cutoff)+ (delivery,) = evaluation.deliveries++ with patch("products.alerts_platform.backend.delivery.email.EmailMessage") as email_message:+ sent = platform_testing.deliver(delivery)++ assert sent == 1+ assert delivery.configuration_id == str(configuration_id)+ assert "Sync failures" in email_message.call_args.kwargs["subject"]+ assert [call.kwargs["email"] for call in email_message.return_value.add_recipient.call_args_list] == sorted(+ [self.user.email, "[email protected]"]+ )+ email_message.return_value.send.assert_called_once_with()