Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(workflows): add a read-only email domain status endpoint (#120402)

Meikel Ratz0a33e3a9c40c
diff --git a/posthog/api/integration.py b/posthog/api/integration.py
index 1d836e2ae4fe..b37015faca32 100644
--- a/posthog/api/integration.py
+++ b/posthog/api/integration.py
@@ -20,11 +20,11 @@ import structlog
from django_filters.rest_framework import DjangoFilterBackend
from django_redis.cache import RedisCache
from django_redis.exceptions import ConnectionInterrupted
-from drf_spectacular.utils import extend_schema, extend_schema_field, extend_schema_serializer
+from drf_spectacular.utils import OpenApiResponse, extend_schema, extend_schema_field, extend_schema_serializer
from prometheus_client import Counter
from redis.exceptions import RedisError
from rest_framework import mixins, serializers, status, viewsets
-from rest_framework.exceptions import APIException, PermissionDenied, Throttled, ValidationError
+from rest_framework.exceptions import APIException, NotFound, PermissionDenied, Throttled, ValidationError
from rest_framework.permissions import BasePermission, IsAuthenticated
from rest_framework.request import Request
from rest_framework.response import Response
@@ -45,10 +45,16 @@ from posthog.api.github_callback.types import (
github_app_install_url,
is_valid_github_installation_id,
)
+from posthog.api.integration_email_status_serializers import (
+ EmailDomainStatusQuerySerializer,
+ EmailDomainStatusSerializer,
+ EmailDomainVerificationSerializer,
+)
+from posthog.api.mixins import ValidatedRequest, validated_request
from posthog.api.routing import TeamAndOrgViewSetMixin
from posthog.api.shared import UserBasicSerializer
from posthog.api.utils import action
-from posthog.auth import SessionAuthentication
+from posthog.auth import OAuthAccessTokenAuthentication, PersonalAPIKeyAuthentication, SessionAuthentication
from posthog.domain_connect import discover_domain_connect, extract_root_domain_and_host, get_available_providers
from posthog.egress.github.transport import GitHubRateLimitError
from posthog.egress.limiter.policies import Priority
@@ -120,6 +126,7 @@ from posthog.permissions import (
TeamMemberStrictManagementPermission,
TimeSensitiveActionPermission,
)
+from posthog.ph_client import feature_enabled_or_false
from posthog.rate_limit import GitHubRepositoryRefreshThrottle
from posthog.tasks.email import send_integration_access_request
from posthog.utils import absolute_uri, is_relative_url
@@ -131,6 +138,7 @@ from products.slack_app.backend.services.slack_auth import SLACK_AUTH_FAILURE_CO
from products.tasks.backend.facade.api import get_in_progress_runs_for_github_integration
from products.tasks.backend.facade.contracts import InProgressGithubRunsDTO
from products.workflows.backend.facade.api import get_active_workflows_using_integration
+from products.workflows.backend.facade.contracts import EmailProviderUnavailableError
logger = structlog.get_logger(__name__)
@@ -147,6 +155,13 @@ stripe_marketplace_install_counter = Counter(
GITHUB_REPOSITORY_NAME_RE = re.compile(r"[A-Za-z0-9_.\-]+")
+class EmailProviderUnavailable(APIException):
+ # A stable code lets the setup page keep polling through an outage instead of treating it as a broken sender.
+ status_code = status.HTTP_503_SERVICE_UNAVAILABLE
+ default_code = "email_provider_unavailable"
+ default_detail = "Couldn't reach the email provider to check this domain. Try again in a moment."
+
+
class SlackIntegrationInactiveError(APIException):
# Actionable 4xx (reconnect Slack) rather than a raw 500, so the channel picker can render
# guidance inline instead of an unhelpful "server error" toast on every retry.
@@ -1488,6 +1503,7 @@ class IntegrationViewSet(
"anthropic_managed_agents",
"anthropic_managed_agent_environments",
"anthropic_managed_agent_vaults",
+ "email_status",
]
scope_object_write_actions = [
"create",
@@ -1499,6 +1515,7 @@ class IntegrationViewSet(
"github_prepare_callback",
"github_link_existing",
"github_oauth_authorize",
+ "email_verify",
# Side-effecting POST (emails admins) — a read-only token must not be able to trigger it.
"request_access",
]
@@ -2650,12 +2667,79 @@ class IntegrationViewSet(
search = query_serializer.validated_data["search"]
return assignees_response(lambda: GitLabIntegration(instance).list_assignees(search))
+ @extend_schema(
+ request=None,
+ responses={200: EmailDomainVerificationSerializer},
+ summary="Verify an email sending domain",
+ description="Ask SES to check the domain's DNS records again. When SES has verified the domain, mark "
+ "the domain's senders as verified so they can send. Call it once email/status reports status verified.",
+ )
@action(methods=["POST"], detail=True, url_path="email/verify")
def email_verify(self, request, **kwargs):
- email = EmailIntegration(self.get_object())
- verification_result = email.verify()
+ instance = self.get_object()
+ email = EmailIntegration(instance)
+ verification_result = email.verify(
+ on_domain_verified=lambda: self._report_email_domain_verified(request, instance)
+ )
return Response(verification_result)
+ @validated_request(
+ query_serializer=EmailDomainStatusQuerySerializer,
+ responses={
+ 200: EmailDomainStatusSerializer,
+ 503: OpenApiResponse(
+ description="The email provider could not be reached. The error code is email_provider_unavailable. "
+ "Nothing changed, so poll again."
+ ),
+ },
+ summary="Get email sending domain status",
+ description="Read the setup progress of an email sender's domain from SES and public DNS without "
+ "changing either. Poll this until status is verified, then call email/verify once so the sender can send.",
+ )
+ @action(methods=["GET"], detail=True, url_path="email/status")
+ def email_status(self, request: ValidatedRequest, **kwargs: Any) -> Response:
+ if not any(
+ feature_enabled_or_false(
+ flag,
+ str(cast(User, request.user).distinct_id),
+ groups={"organization": str(self.team.organization_id), "project": str(self.team.uuid)},
+ group_properties={
+ "organization": {"id": str(self.team.organization_id)},
+ "project": {"id": str(self.team.uuid)},
+ },
+ send_feature_flag_events=False,
+ )
+ for flag in ("workflows-email-domain-wizard", "workflows-email-domain-agent-setup")
+ ):
+ raise NotFound()
+ instance = self.get_object()
+ if instance.kind != "email":
+ raise ValidationError("email/status is only supported for email integrations")
+ try:
+ report = EmailIntegration(instance).status(
+ refresh=request.validated_query_data["refresh"],
+ )
+ except EmailProviderUnavailableError as error:
+ raise EmailProviderUnavailable() from error
+ return Response(EmailDomainStatusSerializer(report).data)
+
+ def _report_email_domain_verified(self, request: Request, instance: Integration) -> None:
+ report_user_action(
+ request.user,
+ "email domain verified",
+ {
+ "method": self._email_domain_setup_method(request, instance),
+ "seconds_since_created": int((timezone.now() - instance.created_at).total_seconds()),
+ },
+ team=self.team,
+ )
+
+ @staticmethod
+ def _email_domain_setup_method(request: Request, instance: Integration) -> str:
+ if isinstance(request.successful_authenticator, PersonalAPIKeyAuthentication | OAuthAccessTokenAuthentication):
+ return "agent"
+ return instance.config.get("setup_method", "manual")
+
@extend_schema(responses={200: IntegrationSerializer})
@action(methods=["PATCH"], detail=True, url_path="email")
def email_update(self, request, **kwargs) -> Response:
diff --git a/posthog/api/integration_email_status_serializers.py b/posthog/api/integration_email_status_serializers.py
new file mode 100644
index 000000000000..3658d5c83c8e
--- /dev/null
+++ b/posthog/api/integration_email_status_serializers.py
@@ -0,0 +1,97 @@
+from django.db import models
+
+from rest_framework import serializers
+
+from products.workflows.backend.facade.enums import (
+ EmailDomainSetupRecordKind,
+ EmailDomainSetupRecordStatus,
+ EmailDomainSetupRecordType,
+ EmailDomainSetupStatus,
+ EmailDomainSetupStepKey,
+ EmailDomainSetupStepState,
+)
+
+
+class EmailDomainStatusQuerySerializer(serializers.Serializer):
+ refresh = serializers.BooleanField(
+ default=False,
+ help_text="Skip the 10 second status cache and read SES and DNS again. Use it after changing DNS records.",
+ )
+
+
+class EmailDomainStatusStepSerializer(serializers.Serializer):
+ key = serializers.ChoiceField(
+ choices=EmailDomainSetupStepKey.choices,
+ help_text="The setup step: domain_added (SES knows the domain), records_found (every DNS record is "
+ "published), verified (SES verified the domain and the sender can send).",
+ )
+ state = serializers.ChoiceField(choices=EmailDomainSetupStepState.choices, help_text="Progress of this step.")
+
+
+class EmailDomainStatusRecordSerializer(serializers.Serializer):
+ kind = serializers.ChoiceField(
+ choices=EmailDomainSetupRecordKind.choices,
+ help_text="What the record does: verification proves domain ownership, dkim signs email, spf and "
+ "mail_from_spf list SES as an allowed sender, mail_from_mx routes bounces, dmarc sets the policy for "
+ "unauthenticated mail.",
+ )
+ hostname = serializers.CharField(help_text="Full DNS name to create the record at, including the domain.")
+ type = serializers.ChoiceField(choices=EmailDomainSetupRecordType.choices, help_text="DNS record type.")
+ value = serializers.CharField(
+ help_text="Value to publish. For an SPF record, add the include to an existing SPF record instead of "
+ "creating a second one. For DMARC, any existing v=DMARC1 record counts."
+ )
+ priority = serializers.IntegerField(allow_null=True, help_text="MX priority. Null for other record types.")
+ status = serializers.ChoiceField(
+ choices=EmailDomainSetupRecordStatus.choices,
+ help_text="pending: not found in DNS yet. found: published in DNS, SES has not confirmed it yet. "
+ "verified: SES confirmed the record, or for DMARC a v=DMARC1 record exists.",
+ )
+
+
+class EmailDomainStatusSerializer(serializers.Serializer):
+ status = serializers.ChoiceField(
+ source="check.status",
+ choices=EmailDomainSetupStatus.choices,
+ help_text="Overall setup status. records_found means every record is published and SES has not "
+ "confirmed yet. temporary_failure means SES could not see the records, usually because DNS changes "
+ "have not spread yet. failed means SES gave up after 72 hours; start verification again.",
+ )
+ verified = serializers.BooleanField(
+ help_text="Whether the sender can send email. Set by email/verify once SES verifies the domain."
+ )
+ checked_at = serializers.DateTimeField(
+ help_text="When SES and DNS were read. Responses are cached for 10 seconds unless refresh is set."
+ )
+ steps = EmailDomainStatusStepSerializer(
+ source="check.steps", many=True, help_text="The three setup steps, in order."
+ )
+ records = EmailDomainStatusRecordSerializer(
+ source="check.records",
+ many=True,
+ help_text="DNS records to publish at the domain's DNS host, with the status of each. Empty when "
+ "status is not_started.",
+ )
+
+
+class EmailDomainVerificationStatus(models.TextChoices):
+ SUCCESS = "success"
+ PENDING = "pending"
+ FAILED = "failed"
+
+
+class EmailDomainVerificationRecordSerializer(serializers.Serializer):
+ type = serializers.CharField(help_text="What the record does: verification, dkim, mail_from or dmarc.")
+ recordType = serializers.CharField(help_text="DNS record type: TXT, CNAME or MX.")
+ recordHostname = serializers.CharField(help_text="DNS name to create the record at.")
+ recordValue = serializers.CharField(help_text="Value to publish.")
+ status = serializers.CharField(help_text="success once SES or DNS confirms the record, else pending.")
+ priority = serializers.IntegerField(required=False, help_text="MX priority. Only set on MX records.")
+
+
+class EmailDomainVerificationSerializer(serializers.Serializer):
+ status = serializers.ChoiceField(
+ choices=EmailDomainVerificationStatus.choices,
+ help_text="success when SES verified the domain and its senders are now marked verified.",
+ )
+ dnsRecords = EmailDomainVerificationRecordSerializer(many=True, help_text="DNS records the domain needs.")
diff --git a/posthog/models/integration/email.py b/posthog/models/integration/email.py
index 2dc30b43d6cc..a03a0223262b 100644
--- a/posthog/models/integration/email.py
+++ b/posthog/models/integration/email.py
@@ -1,15 +1,23 @@
"""Native email-sending integration (SES / maildev) and its cleanup signal."""
+import json
+from collections.abc import Callable
+from datetime import datetime
from typing import TYPE_CHECKING, Any
from django.conf import settings
+from django.core.cache import cache
from django.db import models, transaction
from django.dispatch import receiver
+from django.utils import timezone
+import structlog
from disposable_email_domains import blocklist as disposable_email_domains_list
from free_email_domains import whitelist as free_email_domains_list
from rest_framework.exceptions import ValidationError
+from posthog.dataclasses import frozen
+from posthog.exceptions_capture import capture_exception
from posthog.models.team.team import Team
from posthog.models.user import User
from posthog.plugins.plugin_server_api import reload_integrations_on_workers
@@ -17,7 +25,18 @@ from posthog.plugins.plugin_server_api import reload_integrations_on_workers
from . import model
if TYPE_CHECKING:
- from products.workflows.backend.facade.contracts import EmailDomainVerification
+ from products.workflows.backend.facade.contracts import EmailDomainCheck, EmailDomainVerification
+
+logger = structlog.get_logger(__name__)
+
+EMAIL_DOMAIN_STATUS_CACHE_SECONDS = 10
+
+
+@frozen
+class EmailDomainStatusReport:
+ check: "EmailDomainCheck"
+ verified: bool
+ checked_at: datetime
class EmailIntegration:
@@ -125,7 +144,7 @@ class EmailIntegration:
return self.integration
- def verify(self) -> "EmailDomainVerification":
+ def verify(self, *, on_domain_verified: Callable[[], None] | None = None) -> "EmailDomainVerification":
domain = self.integration.config.get("domain")
provider = self.integration.config.get("provider", "ses")
mail_from_subdomain = self.integration.config.get("mail_from_subdomain", "feedback")
@@ -153,28 +172,110 @@ class EmailIntegration:
raise ValueError(f"Invalid provider: {provider}")
if verification_result.get("status") == "success":
- # We can validate all other integrations with the same domain and provider
- all_integrations_for_domain = model.Integration.objects.filter(
- team_id=self.integration.team_id,
- kind="email",
- config__domain=domain,
- config__provider=provider,
+ self._mark_domain_verified(on_domain_verified)
+
+ return verification_result
+
+ def status(
+ self,
+ *,
+ refresh: bool,
+ ) -> EmailDomainStatusReport:
+ from products.workflows.backend.facade.contracts import (
+ EmailDomainCheck, # noqa: PLC0415 — keeps the workflows facade off the model import path
+ )
+
+ cache_key = f"email_domain_status:{self.integration.id}"
+ if not refresh and isinstance(cached := self._cached_status(cache_key), str):
+ entry = json.loads(cached)
+ return EmailDomainStatusReport(
+ check=EmailDomainCheck.from_json(entry["check"]),
+ verified=bool(entry["verified"]),
+ checked_at=datetime.fromisoformat(entry["checked_at"]),
)
- for integration in all_integrations_for_domain:
- integration.config["verified"] = True
- integration.save()
- reload_integrations_on_workers(
- self.integration.team_id, [integration.id for integration in all_integrations_for_domain]
+ check = self._check_domain()
+ # Reading the status never saves the verification. The caller records it with verify().
+ unrecorded_verification = check.is_verified and not self.integration.config.get("verified")
+
+ report = EmailDomainStatusReport(
+ check=check, verified=bool(self.integration.config.get("verified")), checked_at=timezone.now()
+ )
+ if check.every_dns_lookup_answered and not unrecorded_verification:
+ entry = {"check": check.to_json(), "verified": report.verified, "checked_at": report.checked_at.isoformat()}
+ try:
+ cache.set(cache_key, json.dumps(entry), EMAIL_DOMAIN_STATUS_CACHE_SECONDS)
+ except Exception:
+ logger.warning("email_domain_status_cache_write_failed", exc_info=True)
+ return report
+
+ # The cache only spares SES and DNS reads, so a cache outage falls back to a live check instead of failing.
+ @staticmethod
+ def _cached_status(cache_key: str) -> object:
+ try:
+ return cache.get(cache_key)
+ except Exception:
+ logger.warning("email_domain_status_cache_read_failed", exc_info=True)
+ return None
+
+ def _check_domain(self) -> "EmailDomainCheck":
+ domain = self.integration.config["domain"]
+ mail_from_subdomain = self.integration.config.get("mail_from_subdomain", "feedback")
+ provider = self.integration.config.get("provider", "ses")
+
+ if provider == "ses":
+ from products.workflows.backend.facade.api import (
+ get_ses_email_domain_status, # noqa: PLC0415 — keeps the workflows facade off the model import path
)
+ return get_ses_email_domain_status(
+ domain, mail_from_subdomain=mail_from_subdomain, team_id=self.integration.team_id
+ )
+ if provider == "maildev":
from products.workflows.backend.facade.api import (
- set_default_email_sender_if_unset, # noqa: PLC0415 — keeps the workflows facade off the model import path
+ get_maildev_email_domain_status, # noqa: PLC0415 — keeps the workflows facade off the model import path
)
- set_default_email_sender_if_unset(self.integration.team_id, self.integration.id)
+ return get_maildev_email_domain_status(domain, mail_from_subdomain=mail_from_subdomain)
+ raise ValueError(f"Invalid provider: {provider}")
+
+ def _mark_domain_verified(self, on_domain_verified: Callable[[], None] | None) -> None:
+ with transaction.atomic():
+ senders = list(
+ model.Integration.objects.select_for_update()
+ .filter(
+ team_id=self.integration.team_id,
+ kind="email",
+ config__domain=self.integration.config.get("domain"),
+ config__provider=self.integration.config.get("provider", "ses"),
+ )
+ .order_by("id")
+ )
+ unverified_senders = [sender for sender in senders if not sender.config.get("verified")]
+ for sender in unverified_senders:
+ sender.config["verified"] = True
+ sender.save()
- return verification_result
+ self.integration.config["verified"] = True
+
+ from products.workflows.backend.facade.api import (
+ set_default_email_sender_if_unset, # noqa: PLC0415 — keeps the workflows facade off the model import path
+ )
+
+ set_default_email_sender_if_unset(self.integration.team_id, self.integration.id)
+
+ # A sender added to a domain that is already verified joins it, so only the first verification counts.
+ if on_domain_verified and len(unverified_senders) == len(senders):
+ on_domain_verified()
+ self._reload_senders_on_workers(senders)
+
+ # Workers also refresh integrations on their own schedule, so a failed reload only delays sending.
+ # Failing the request instead would hide a verification that is already saved.
+ def _reload_senders_on_workers(self, senders: list[model.Integration]) -> None:
+ try:
+ reload_integrations_on_workers(self.integration.team_id, [sender.id for sender in senders])
+ except Exception as error:
+ capture_exception(error, {"team_id": self.integration.team_id})
@receiver(models.signals.post_delete, sender=model.Integration)
diff --git a/products/integrations/frontend/generated/api.schemas.ts b/products/integrations/frontend/generated/api.schemas.ts
index 30eda29b193d..d3bdeea6b507 100644
--- a/products/integrations/frontend/generated/api.schemas.ts
+++ b/products/integrations/frontend/generated/api.schemas.ts
@@ -348,6 +348,210 @@ export interface PatchedIntegrationConfigApi {
readonly installation_status?: InstallationStatusEnumApi | null
}
+/**
+ * * `not_started` - Not started
+ * * `pending` - Waiting for DNS records
+ * * `records_found` - Records found, waiting for verification
+ * * `verified` - Verified
+ * * `temporary_failure` - Temporary failure
+ * * `failed` - Failed
+ */
+export type EmailDomainSetupStatusEnumApi =
+ (typeof EmailDomainSetupStatusEnumApi)[keyof typeof EmailDomainSetupStatusEnumApi]
+
+export const EmailDomainSetupStatusEnumApi = {
+ NotStarted: 'not_started',
+ Pending: 'pending',
+ RecordsFound: 'records_found',
+ Verified: 'verified',
+ TemporaryFailure: 'temporary_failure',
+ Failed: 'failed',
+} as const
+
+/**
+ * * `domain_added` - Domain added
+ * * `records_found` - DNS records found
+ * * `verified` - Verified
+ */
+export type EmailDomainSetupStepKeyEnumApi =
+ (typeof EmailDomainSetupStepKeyEnumApi)[keyof typeof EmailDomainSetupStepKeyEnumApi]
+
+export const EmailDomainSetupStepKeyEnumApi = {
+ DomainAdded: 'domain_added',
+ RecordsFound: 'records_found',
+ Verified: 'verified',
+} as const
+
+/**
+ * * `done` - Done
+ * * `pending` - Pending
+ * * `failed` - Failed
+ */
+export type EmailDomainSetupStepStateEnumApi =
+ (typeof EmailDomainSetupStepStateEnumApi)[keyof typeof EmailDomainSetupStepStateEnumApi]
+
+export const EmailDomainSetupStepStateEnumApi = {
+ Done: 'done',
+ Pending: 'pending',
+ Failed: 'failed',
+} as const
+
+export interface EmailDomainStatusStepApi {
+ /** The setup step: domain_added (SES knows the domain), records_found (every DNS record is published), verified (SES verified the domain and the sender can send).
+ *
+ * * `domain_added` - Domain added
+ * * `records_found` - DNS records found
+ * * `verified` - Verified */
+ key: EmailDomainSetupStepKeyEnumApi
+ /** Progress of this step.
+ *
+ * * `done` - Done
+ * * `pending` - Pending
+ * * `failed` - Failed */
+ state: EmailDomainSetupStepStateEnumApi
+}
+
+/**
+ * * `verification` - Domain ownership
+ * * `dkim` - DKIM signing
+ * * `spf` - SPF
+ * * `mail_from_mx` - MAIL FROM MX
+ * * `mail_from_spf` - MAIL FROM SPF
+ * * `dmarc` - DMARC
+ */
+export type EmailDomainSetupRecordKindEnumApi =
+ (typeof EmailDomainSetupRecordKindEnumApi)[keyof typeof EmailDomainSetupRecordKindEnumApi]
+
+export const EmailDomainSetupRecordKindEnumApi = {
+ Verification: 'verification',
+ Dkim: 'dkim',
+ Spf: 'spf',
+ MailFromMx: 'mail_from_mx',
+ MailFromSpf: 'mail_from_spf',
+ Dmarc: 'dmarc',
+} as const
+
+/**
+ * * `TXT` - TXT
+ * * `CNAME` - CNAME
+ * * `MX` - MX
+ */
+export type EmailDomainSetupRecordTypeEnumApi =
+ (typeof EmailDomainSetupRecordTypeEnumApi)[keyof typeof EmailDomainSetupRecordTypeEnumApi]
+
+export const EmailDomainSetupRecordTypeEnumApi = {
+ Txt: 'TXT',
+ Cname: 'CNAME',
+ Mx: 'MX',
+} as const
+
+/**
+ * * `pending` - Not found yet
+ * * `found` - Found in DNS
+ * * `verified` - Verified
+ */
+export type EmailDomainSetupRecordStatusEnumApi =
+ (typeof EmailDomainSetupRecordStatusEnumApi)[keyof typeof EmailDomainSetupRecordStatusEnumApi]
+
+export const EmailDomainSetupRecordStatusEnumApi = {
+ Pending: 'pending',
+ Found: 'found',
+ Verified: 'verified',
+} as const
+
+export interface EmailDomainStatusRecordApi {
+ /** What the record does: verification proves domain ownership, dkim signs email, spf and mail_from_spf list SES as an allowed sender, mail_from_mx routes bounces, dmarc sets the policy for unauthenticated mail.
+ *
+ * * `verification` - Domain ownership
+ * * `dkim` - DKIM signing
+ * * `spf` - SPF
+ * * `mail_from_mx` - MAIL FROM MX
+ * * `mail_from_spf` - MAIL FROM SPF
+ * * `dmarc` - DMARC */
+ kind: EmailDomainSetupRecordKindEnumApi
+ /** Full DNS name to create the record at, including the domain. */
+ hostname: string
+ /** DNS record type.
+ *
+ * * `TXT` - TXT
+ * * `CNAME` - CNAME
+ * * `MX` - MX */
+ type: EmailDomainSetupRecordTypeEnumApi
+ /** Value to publish. For an SPF record, add the include to an existing SPF record instead of creating a second one. For DMARC, any existing v=DMARC1 record counts. */
+ value: string
+ /**
+ * MX priority. Null for other record types.
+ * @nullable
+ */
+ priority: number | null
+ /** pending: not found in DNS yet. found: published in DNS, SES has not confirmed it yet. verified: SES confirmed the record, or for DMARC a v=DMARC1 record exists.
+ *
+ * * `pending` - Not found yet
+ * * `found` - Found in DNS
+ * * `verified` - Verified */
+ status: EmailDomainSetupRecordStatusEnumApi
+}
+
+export interface EmailDomainStatusApi {
+ /** Overall setup status. records_found means every record is published and SES has not confirmed yet. temporary_failure means SES could not see the records, usually because DNS changes have not spread yet. failed means SES gave up after 72 hours; start verification again.
+ *
+ * * `not_started` - Not started
+ * * `pending` - Waiting for DNS records
+ * * `records_found` - Records found, waiting for verification
+ * * `verified` - Verified
+ * * `temporary_failure` - Temporary failure
+ * * `failed` - Failed */
+ status: EmailDomainSetupStatusEnumApi
+ /** Whether the sender can send email. Set by email/verify once SES verifies the domain. */
+ verified: boolean
+ /** When SES and DNS were read. Responses are cached for 10 seconds unless refresh is set. */
+ checked_at: string
+ /** The three setup steps, in order. */
+ steps: EmailDomainStatusStepApi[]
+ /** DNS records to publish at the domain's DNS host, with the status of each. Empty when status is not_started. */
+ records: EmailDomainStatusRecordApi[]
+}
+
+/**
+ * * `success` - Success
+ * * `pending` - Pending
+ * * `failed` - Failed
+ */
+export type EmailDomainVerificationStatusEnumApi =
+ (typeof EmailDomainVerificationStatusEnumApi)[keyof typeof EmailDomainVerificationStatusEnumApi]
+
+export const EmailDomainVerificationStatusEnumApi = {
+ Success: 'success',
+ Pending: 'pending',
+ Failed: 'failed',
+} as const
+
+export interface EmailDomainVerificationRecordApi {
+ /** What the record does: verification, dkim, mail_from or dmarc. */
+ type: string
+ /** DNS record type: TXT, CNAME or MX. */
+ recordType: string
+ /** DNS name to create the record at. */
+ recordHostname: string
+ /** Value to publish. */
+ recordValue: string
+ /** success once SES or DNS confirms the record, else pending. */
+ status: string
+ /** MX priority. Only set on MX records. */
+ priority?: number
+}
+
+export interface EmailDomainVerificationApi {
+ /** success when SES verified the domain and its senders are now marked verified.
+ *
+ * * `success` - Success
+ * * `pending` - Pending
+ * * `failed` - Failed */
+ status: EmailDomainVerificationStatusEnumApi
+ /** DNS records the domain needs. */
+ dnsRecords: EmailDomainVerificationRecordApi[]
+}
+
export interface IntegrationAssigneeApi {
/** Provider user identifier to pass as error tracking config.assignee: a Linear user ID, a GitHub login, a GitLab user ID, or a Jira account ID. */
id: string
@@ -904,6 +1108,13 @@ export type IntegrationsChannelsRetrieveParams = {
search?: string
}
+export type IntegrationsEmailStatusRetrieveParams = {
+ /**
+ * Skip the 10 second status cache and read SES and DNS again. Use it after changing DNS records.
+ */
+ refresh?: boolean
+}
+
export type IntegrationsGithubAssigneesRetrieveParams = {
/**
* Repository name, or owner/name, whose assignable users to list.
diff --git a/products/integrations/frontend/generated/api.ts b/products/integrations/frontend/generated/api.ts
index 54bdf2e0adc4..779ae35acda3 100644
--- a/products/integrations/frontend/generated/api.ts
+++ b/products/integrations/frontend/generated/api.ts
@@ -9,6 +9,8 @@ import { apiMutator } from '../../../../frontend/src/lib/api-orval-mutator'
* OpenAPI spec version: 1.0.0
*/
import type {
+ EmailDomainStatusApi,
+ EmailDomainVerificationApi,
GitHubAvailableInstallationsResponseApi,
GitHubBranchesResponseApi,
GitHubLinkExistingRequestApi,
@@ -23,6 +25,7 @@ import type {
IntegrationAssigneesResponseApi,
IntegrationConfigApi,
IntegrationsChannelsRetrieveParams,
+ IntegrationsEmailStatusRetrieveParams,
IntegrationsGithubAssigneesRetrieveParams,
IntegrationsGithubBranchesRetrieveParams,
IntegrationsGithubReposRetrieveParams,
@@ -397,21 +400,58 @@ export const integrationsEmailPartialUpdate = async (
})
}
+export const getIntegrationsEmailStatusRetrieveUrl = (
+ projectId: string,
+ id: number,
+ params?: IntegrationsEmailStatusRetrieveParams
+) => {
+ const normalizedParams = new URLSearchParams()
+
+ Object.entries(params || {}).forEach(([key, value]) => {
+ if (value !== undefined) {
+ normalizedParams.append(key, value === null ? 'null' : String(value))
+ }
+ })
+
+ const stringifiedParams = normalizedParams.toString()
+
+ return stringifiedParams.length > 0
+ ? `/api/projects/${projectId}/integrations/${id}/email/status/?${stringifiedParams}`
+ : `/api/projects/${projectId}/integrations/${id}/email/status/`
+}
+
+/**
+ * Read the setup progress of an email sender's domain from SES and public DNS without changing either. Poll this until status is verified, then call email/verify once so the sender can send.
+ * @summary Get email sending domain status
+ */
+export const integrationsEmailStatusRetrieve = async (
+ projectId: string,
+ id: number,
+ params?: IntegrationsEmailStatusRetrieveParams,
+ options?: RequestInit
+): Promise<EmailDomainStatusApi> => {
+ return apiMutator<EmailDomainStatusApi>(getIntegrationsEmailStatusRetrieveUrl(projectId, id, params), {
+ ...options,
+ method: 'GET',
+ })
+}
+
export const getIntegrationsEmailVerifyCreateUrl = (projectId: string, id: number) => {
return `/api/projects/${projectId}/integrations/${id}/email/verify/`
}
+/**
+ * Ask SES to check the domain's DNS records again. When SES has verified the domain, mark the domain's senders as verified so they can send. Call it once email/status reports status verified.
+ * @summary Verify an email sending domain
+ */
export const integrationsEmailVerifyCreate = async (
projectId: string,
id: number,
- integrationConfigApi: NonReadonly<IntegrationConfigApi>,
options?: RequestInit
-): Promise<void> => {
- return apiMutator<void>(getIntegrationsEmailVerifyCreateUrl(projectId, id), {
+): Promise<EmailDomainVerificationApi> => {
+ return apiMutator<EmailDomainVerificationApi>(getIntegrationsEmailVerifyCreateUrl(projectId, id), {
...options,
method: 'POST',
- headers: { 'Content-Type': 'application/json', ...options?.headers },
- body: JSON.stringify(integrationConfigApi),
})
}
diff --git a/products/integrations/frontend/generated/api.zod.ts b/products/integrations/frontend/generated/api.zod.ts
index a4e1ca6b826c..97749d591c46 100644
--- a/products/integrations/frontend/generated/api.zod.ts
+++ b/products/integrations/frontend/generated/api.zod.ts
@@ -183,68 +183,6 @@ export const IntegrationsEmailPartialUpdateBody = /* @__PURE__ */ zod
})
.describe('Standard Integration serializer.')
-export const IntegrationsEmailVerifyCreateBody = /* @__PURE__ */ zod
- .object({
- kind: zod
- .enum([
- 'anthropic',
- 'apple-ads',
- 'apns',
- 'aws-redshift',
- 'aws-s3',
- 'azure-blob',
- 'bing-ads',
- 'clickup',
- 'customerio-app',
- 'customerio-track',
- 'customerio-webhook',
- 'databricks',
- 'email',
- 'firebase',
- 'github',
- 'gitlab',
- 'google-ads',
- 'google-analytics',
- 'google-calendar',
- 'google-cloud-service-account',
- 'google-cloud-storage',
- 'google-pubsub',
- 'google-search-console',
- 'google-sheets',
- 'helpscout',
- 'hubspot',
- 'instagram',
- 'intercom',
- 'jira',
- 'linear',
- 'linkedin-ads',
- 'meta-ads',
- 'pardot',
- 'pinterest-ads',
- 'postgresql',
- 'posthog',
- 'reddit-ads',
- 'twitter-ads',
- 'resend',
- 's3-compatible',
- 'salesforce',
- 'slack',
- 'slack-posthog-code',
- 'snapchat',
- 'snowflake',
- 'stripe',
- 'tiktok-ads',
- 'twilio',
- 'vercel',
- 'youtube-analytics',
- ])
- .describe(
- '\* `anthropic` - Anthropic\n\* `apple-ads` - Apple Ads\n\* `apns` - Apple Push\n\* `aws-redshift` - Aws Redshift\n\* `aws-s3` - Aws S3\n\* `azure-blob` - Azure Blob\n\* `bing-ads` - Bing Ads\n\* `clickup` - Clickup\n\* `customerio-app` - Customerio App\n\* `customerio-track` - Customerio Track\n\* `customerio-webhook` - Customerio Webhook\n\* `databricks` - Databricks\n\* `email` - Email\n\* `firebase` - Firebase\n\* `github` - Github\n\* `gitlab` - Gitlab\n\* `google-ads` - Google Ads\n\* `google-analytics` - Google Analytics\n\* `google-calendar` - Google Calendar\n\* `google-cloud-service-account` - Google Cloud Service Account\n\* `google-cloud-storage` - Google Cloud Storage\n\* `google-pubsub` - Google Pubsub\n\* `google-search-console` - Google Search Console\n\* `google-sheets` - Google Sheets\n\* `helpscout` - Helpscout\n\* `hubspot` - Hubspot\n\* `instagram` - Instagram\n\* `intercom` - Intercom\n\* `jira` - Jira\n\* `linear` - Linear\n\* `linkedin-ads` - Linkedin Ads\n\* `meta-ads` - Meta Ads\n\* `pardot` - Pardot\n\* `pinterest-ads` - Pinterest Ads\n\* `postgresql` - Postgresql\n\* `posthog` - Posthog\n\* `reddit-ads` - Reddit Ads\n\* `twitter-ads` - Twitter Ads\n\* `resend` - Resend\n\* `s3-compatible` - S3 Compatible\n\* `salesforce` - Salesforce\n\* `slack` - Slack\n\* `slack-posthog-code` - Slack Posthog Code\n\* `snapchat` - Snapchat\n\* `snowflake` - Snowflake\n\* `stripe` - Stripe\n\* `tiktok-ads` - Tiktok Ads\n\* `twilio` - Twilio\n\* `vercel` - Vercel\n\* `youtube-analytics` - Youtube Analytics'
- ),
- config: zod.unknown().optional(),
- })
- .describe('Standard Integration serializer.')
-
/**
* Unified endpoint for generating Domain Connect apply URLs.
*
diff --git a/products/integrations/mcp/tools.yaml b/products/integrations/mcp/tools.yaml
index d92d6daae08f..dda17165884f 100644
--- a/products/integrations/mcp/tools.yaml
+++ b/products/integrations/mcp/tools.yaml
@@ -59,6 +59,38 @@ tools:
to find a channel_id when wiring up Slack delivery for an alert — pass the channel id into the
cdp-functions-create inputs.channel.value field. Requires the Slack integration's id; use integrations-list
(filter by kind=slack) to find it.
+ integrations-email-status-retrieve:
+ operation: integrations_email_status_retrieve
+ enabled: true
+ scopes:
+ - integration:read
+ annotations:
+ readOnly: true
+ destructive: false
+ idempotent: true
+ title: Get email sending domain status
+ description: >
+ Get the setup progress of an email sender's domain by integration ID. Returns the overall status, the three
+ setup steps, and every DNS record to publish with its current status (pending, found, verified). Poll it
+ after adding DNS records until status is verified, waiting at least 10 seconds between calls. Pass
+ refresh=true right after changing DNS to skip the 10 second cache. It does not change DNS or SES, and it
+ does not mark the sender verified: once status is verified, call integrations-email-verify-create.
+ feature_flag: workflows-email-domain-agent-setup
+ integrations-email-verify-create:
+ operation: integrations_email_verify_create
+ enabled: true
+ scopes:
+ - integration:write
+ annotations:
+ readOnly: false
+ destructive: false
+ idempotent: true
+ title: Verify email sending domain
+ description: >
+ Record that an email sender's domain is verified, by integration ID, so its senders can send. Call it once
+ integrations-email-status-retrieve reports status verified. It asks SES to check the domain again and
+ returns status success when SES has verified it. Needs a project admin.
+ feature_flag: workflows-email-domain-agent-setup
integrations-github-repos-retrieve:
operation: integrations_github_repos_retrieve
enabled: true
diff --git a/products/workflows/backend/facade/api.py b/products/workflows/backend/facade/api.py
index 76fc797a34f0..5a47cc4564c7 100644
--- a/products/workflows/backend/facade/api.py
+++ b/products/workflows/backend/facade/api.py
@@ -10,6 +10,7 @@ from posthog.ingress.contracts import WebhookDelivery
from products.access_control.backend.facade.user_access_control import UserAccessControl
from products.workflows.backend.facade.contracts import (
CohortDependentWorkflow,
+ EmailDomainCheck,
EmailDomainDnsRecord,
EmailDomainVerification,
RecentWorkflow,
@@ -333,6 +334,20 @@ def verify_ses_email_domain(domain: str, *, mail_from_subdomain: str, team_id: i
return providers.SESProvider().verify_email_domain(domain, mail_from_subdomain=mail_from_subdomain, team_id=team_id)
+def get_ses_email_domain_status(domain: str, *, mail_from_subdomain: str, team_id: int) -> EmailDomainCheck:
+ from products.workflows.backend import providers # noqa: PLC0415
+
+ return providers.SESProvider().get_email_domain_status(
+ domain, mail_from_subdomain=mail_from_subdomain, team_id=team_id
+ )
+
+
+def get_maildev_email_domain_status(domain: str, *, mail_from_subdomain: str) -> EmailDomainCheck:
+ from products.workflows.backend import providers # noqa: PLC0415
+
+ return providers.maildev_email_domain_status(domain, mail_from_subdomain=mail_from_subdomain)
+
+
def delete_ses_identity(identity: str) -> None:
from products.workflows.backend import providers # noqa: PLC0415
diff --git a/products/workflows/backend/facade/contracts.py b/products/workflows/backend/facade/contracts.py
index 5143ffb93f5d..40b56ca1fefd 100644
--- a/products/workflows/backend/facade/contracts.py
+++ b/products/workflows/backend/facade/contracts.py
@@ -1,3 +1,4 @@
+import json
from collections.abc import Mapping
from dataclasses import dataclass, field
from datetime import datetime
@@ -7,6 +8,12 @@ from uuid import UUID
from posthog.dataclasses import frozen
from products.workflows.backend.facade.enums import (
+ EmailDomainSetupRecordKind,
+ EmailDomainSetupRecordStatus,
+ EmailDomainSetupRecordType,
+ EmailDomainSetupStatus,
+ EmailDomainSetupStepKey,
+ EmailDomainSetupStepState,
EmailDraftFallbackReason,
EmailDraftOrigin,
HogFlowBatchJobState,
@@ -422,6 +429,86 @@ class EmailDomainVerification(TypedDict):
dnsRecords: list[EmailDomainDnsRecord]
+@frozen
+class EmailDomainStatusRecord:
+ kind: EmailDomainSetupRecordKind
+ hostname: str
+ type: EmailDomainSetupRecordType
+ value: str
+ priority: int | None = None
+ status: EmailDomainSetupRecordStatus = EmailDomainSetupRecordStatus.PENDING
+
+
+@frozen
+class EmailDomainStatusStep:
+ key: EmailDomainSetupStepKey
+ state: EmailDomainSetupStepState
+
+
+@frozen
+class EmailDomainCheck:
+ """What the email provider and public DNS report for a sending domain, read without changing either."""
+
+ status: EmailDomainSetupStatus
+ steps: tuple[EmailDomainStatusStep, ...]
+ records: tuple[EmailDomainStatusRecord, ...]
+ every_dns_lookup_answered: bool = True
+
+ @property
+ def is_verified(self) -> bool:
+ return self.status == EmailDomainSetupStatus.VERIFIED
+
+ # The Django cache pickles what it stores, so the check goes in as a JSON string of plain values.
+ def to_json(self) -> str:
+ return json.dumps(
+ {
+ "status": self.status.value,
+ "steps": [{"key": step.key.value, "state": step.state.value} for step in self.steps],
+ "records": [
+ {
+ "kind": record.kind.value,
+ "hostname": record.hostname,
+ "type": record.type.value,
+ "value": record.value,
+ "priority": record.priority,
+ "status": record.status.value,
+ }
+ for record in self.records
+ ],
+ "every_dns_lookup_answered": self.every_dns_lookup_answered,
+ }
+ )
+
+ @classmethod
+ def from_json(cls, raw: str) -> "EmailDomainCheck":
+ data = json.loads(raw)
+ return cls(
+ status=EmailDomainSetupStatus(data["status"]),
+ steps=tuple(
+ EmailDomainStatusStep(
+ key=EmailDomainSetupStepKey(step["key"]), state=EmailDomainSetupStepState(step["state"])
+ )
+ for step in data["steps"]
+ ),
+ records=tuple(
+ EmailDomainStatusRecord(
+ kind=EmailDomainSetupRecordKind(record["kind"]),
+ hostname=str(record["hostname"]),
+ type=EmailDomainSetupRecordType(record["type"]),
+ value=str(record["value"]),
+ priority=None if record["priority"] is None else int(record["priority"]),
+ status=EmailDomainSetupRecordStatus(record["status"]),
+ )
+ for record in data["records"]
+ ),
+ every_dns_lookup_answered=bool(data["every_dns_lookup_answered"]),
+ )
+
+
+class EmailProviderUnavailableError(Exception):
+ """The email provider could not be reached or refused the read, so the domain's state is unknown."""
+
+
class TwilioPhoneNumber(TypedDict):
"""The fields callers read. The Twilio payload has more keys."""
diff --git a/products/workflows/backend/facade/enums.py b/products/workflows/backend/facade/enums.py
index c10ba1a2b730..50c3715b38b1 100644
--- a/products/workflows/backend/facade/enums.py
+++ b/products/workflows/backend/facade/enums.py
@@ -135,3 +135,45 @@ class RealtimeAudienceCohortState(LabeledStrEnum):
BUILDING = "building", "Building"
REBUILDING = "rebuilding", "Rebuilding"
NEEDS_ATTENTION = "needs_attention", "Needs attention"
+
+
+class EmailDomainSetupStatus(LabeledStrEnum):
+ NOT_STARTED = "not_started", "Not started"
+ PENDING = "pending", "Waiting for DNS records"
+ RECORDS_FOUND = "records_found", "Records found, waiting for verification"
+ VERIFIED = "verified", "Verified"
+ TEMPORARY_FAILURE = "temporary_failure", "Temporary failure"
+ FAILED = "failed", "Failed"
+
+
+class EmailDomainSetupRecordKind(LabeledStrEnum):
+ VERIFICATION = "verification", "Domain ownership"
+ DKIM = "dkim", "DKIM signing"
+ SPF = "spf", "SPF"
+ MAIL_FROM_MX = "mail_from_mx", "MAIL FROM MX"
+ MAIL_FROM_SPF = "mail_from_spf", "MAIL FROM SPF"
+ DMARC = "dmarc", "DMARC"
+
+
+class EmailDomainSetupRecordType(LabeledStrEnum):
+ TXT = "TXT", "TXT"
+ CNAME = "CNAME", "CNAME"
+ MX = "MX", "MX"
+
+
+class EmailDomainSetupRecordStatus(LabeledStrEnum):
+ PENDING = "pending", "Not found yet"
+ FOUND = "found", "Found in DNS"
+ VERIFIED = "verified", "Verified"
+
+
+class EmailDomainSetupStepKey(LabeledStrEnum):
+ DOMAIN_ADDED = "domain_added", "Domain added"
+ RECORDS_FOUND = "records_found", "DNS records found"
+ VERIFIED = "verified", "Verified"
+
+
+class EmailDomainSetupStepState(LabeledStrEnum):
+ DONE = "done", "Done"
+ PENDING = "pending", "Pending"
+ FAILED = "failed", "Failed"
diff --git a/products/workflows/backend/providers/__init__.py b/products/workflows/backend/providers/__init__.py
index 4d32cdad2c54..7aec5e3bc60d 100644
--- a/products/workflows/backend/providers/__init__.py
+++ b/products/workflows/backend/providers/__init__.py
@@ -1,5 +1,5 @@
-from .maildev import MAILDEV_MOCK_DNS_RECORDS
+from .maildev import MAILDEV_MOCK_DNS_RECORDS, maildev_email_domain_status
from .ses import SESProvider
from .twilio import TwilioProvider
-__all__ = ["TwilioProvider", "SESProvider", "MAILDEV_MOCK_DNS_RECORDS"]
+__all__ = ["TwilioProvider", "SESProvider", "MAILDEV_MOCK_DNS_RECORDS", "maildev_email_domain_status"]
diff --git a/products/workflows/backend/providers/maildev.py b/products/workflows/backend/providers/maildev.py
index 2a8025809456..27f63fc697f9 100644
--- a/products/workflows/backend/providers/maildev.py
+++ b/products/workflows/backend/providers/maildev.py
@@ -1,4 +1,8 @@
-from products.workflows.backend.facade.contracts import EmailDomainDnsRecord
+from dataclasses import replace
+
+from products.workflows.backend.facade.contracts import EmailDomainCheck, EmailDomainDnsRecord
+from products.workflows.backend.facade.enums import EmailDomainSetupRecordStatus, EmailDomainSetupStatus
+from products.workflows.backend.services.email_domain_status import build_email_domain_check, build_email_domain_records
MAILDEV_MOCK_DNS_RECORDS: list[EmailDomainDnsRecord] = [
# Mock DNS records for email domain setup when using local maildev
@@ -53,3 +57,17 @@ MAILDEV_MOCK_DNS_RECORDS: list[EmailDomainDnsRecord] = [
"status": "success",
},
]
+
+
+def maildev_email_domain_status(domain: str, *, mail_from_subdomain: str) -> EmailDomainCheck:
+ records = build_email_domain_records(
+ domain=domain,
+ mail_from_subdomain=mail_from_subdomain,
+ verification_token="mock-verification-token",
+ dkim_tokens=["mock1", "mock2", "mock3"],
+ ses_region="us-east-1",
+ )
+ return build_email_domain_check(
+ provider_status=EmailDomainSetupStatus.VERIFIED,
+ records=[replace(record, status=EmailDomainSetupRecordStatus.VERIFIED) for record in records],
+ )
diff --git a/products/workflows/backend/providers/ses.py b/products/workflows/backend/providers/ses.py
index 5173b5324559..4ecb6490fda4 100644
--- a/products/workflows/backend/providers/ses.py
+++ b/products/workflows/backend/providers/ses.py
@@ -2,13 +2,15 @@ import re
import logging
from collections import defaultdict
from collections.abc import Iterable, Iterator, Mapping, Sequence
+from dataclasses import replace
from datetime import UTC, datetime, timedelta
from functools import cached_property
from itertools import batched
from time import monotonic
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, Literal, cast
from django.conf import settings
+from django.core.cache import cache
import boto3
import dns.name
@@ -18,12 +20,23 @@ from botocore.exceptions import BotoCoreError, ClientError
from rest_framework import exceptions
from posthog.dataclasses import frozen
+from posthog.exceptions_capture import capture_exception
from products.workflows.backend.facade.contracts import (
+ EmailDomainCheck,
EmailDomainDnsRecord,
+ EmailDomainStatusRecord,
EmailDomainVerification,
EmailDomainVerificationStatus,
+ EmailProviderUnavailableError,
)
+from products.workflows.backend.facade.enums import (
+ EmailDomainSetupRecordKind,
+ EmailDomainSetupRecordStatus,
+ EmailDomainSetupStatus,
+)
+from products.workflows.backend.services.email_domain_dns import find_published_records
+from products.workflows.backend.services.email_domain_status import build_email_domain_check, build_email_domain_records
if TYPE_CHECKING:
from types_boto3_ses.client import SESClient
@@ -32,6 +45,9 @@ if TYPE_CHECKING:
logger = logging.getLogger(__name__)
+AWS_ACCOUNT_ID_CACHE_KEY = "workflows:ses:aws_account_id"
+AWS_ACCOUNT_ID_CACHE_SECONDS = 24 * 60 * 60
+
# DELIVERY_COMPLAINT is the complaint denominator rather than SEND, because AWS defines it as
# deliveries excluding recipients at ISPs it has no feedback-loop agreement with.
# TRANSIENT_BOUNCE is queried because without it a provider that defers rather than rejects reads
@@ -75,6 +91,8 @@ RECOMMENDATIONS_PAGE_SIZE = 100
# The account findings walk runs in a background task, so it can wait out SES throttling. Adaptive
# mode adds a client-side rate limiter that slows the following pages after a throttle response.
RECOMMENDATIONS_RETRY_CONFIG = Config(retries={"mode": "adaptive", "max_attempts": 8})
+# The status endpoint is polled and holds a request worker, so its reads fail fast and the next poll retries.
+STATUS_READ_CONFIG = Config(connect_timeout=2, read_timeout=5, retries={"mode": "standard", "max_attempts": 2})
@frozen
@@ -276,11 +294,114 @@ def _other_provider_row(isps: Sequence[str], series: IspMetricSeries) -> IspSend
)
+_LegacyRecordPurpose = Literal["verification", "dkim", "mail_from", "dmarc"]
+_LegacyRecordType = Literal["TXT", "CNAME", "MX"]
+
+_LEGACY_RECORD_PURPOSES: dict[EmailDomainSetupRecordKind, _LegacyRecordPurpose] = {
+ EmailDomainSetupRecordKind.VERIFICATION: "verification",
+ EmailDomainSetupRecordKind.SPF: "verification",
+ EmailDomainSetupRecordKind.DKIM: "dkim",
+ EmailDomainSetupRecordKind.MAIL_FROM_MX: "mail_from",
+ EmailDomainSetupRecordKind.MAIL_FROM_SPF: "mail_from",
+ EmailDomainSetupRecordKind.DMARC: "dmarc",
+}
+
+
+def _legacy_dns_record(record: EmailDomainStatusRecord) -> EmailDomainDnsRecord:
+ legacy: EmailDomainDnsRecord = {
+ "type": _LEGACY_RECORD_PURPOSES[record.kind],
+ "recordType": cast(_LegacyRecordType, record.type.value),
+ "recordHostname": "@" if record.kind == EmailDomainSetupRecordKind.SPF else record.hostname,
+ "recordValue": record.value,
+ "status": "pending",
+ }
+ if record.priority is not None:
+ legacy["priority"] = record.priority
+ return legacy
+
+
+_UNKNOWN = "Unknown"
+
+
+@frozen
+class _SESIdentityState:
+ """The SES v1 identity attributes. A status is "Unknown" when SES has no attribute for the domain."""
+
+ verification_status: str
+ verification_token: str | None
+ dkim_status: str
+ dkim_tokens: tuple[str, ...]
+ mail_from_status: str
+ mail_from_domain: str | None
+
+ @classmethod
+ def unknown(cls) -> "_SESIdentityState":
+ return cls(
+ verification_status=_UNKNOWN,
+ verification_token=None,
+ dkim_status=_UNKNOWN,
+ dkim_tokens=(),
+ mail_from_status=_UNKNOWN,
+ mail_from_domain=None,
+ )
+
+ # SES keeps one MAIL FROM domain per identity, and senders on the same domain can ask for
+ # different ones. The status SES reports belongs to whichever was set last.
+ def for_mail_from_domain(self, mail_from_domain: str) -> "_SESIdentityState":
+ if self.mail_from_domain == mail_from_domain:
+ return self
+ return replace(self, mail_from_status=_UNKNOWN)
+
+ @property
+ def statuses(self) -> frozenset[str]:
+ return frozenset((self.verification_status, self.dkim_status, self.mail_from_status))
+
+ def is_ready(self, *, dmarc_published: bool) -> bool:
+ return dmarc_published and all(status == "Success" for status in self.statuses)
+
+ def setup_status(self, *, is_ready: bool) -> EmailDomainSetupStatus:
+ if is_ready:
+ return EmailDomainSetupStatus.VERIFIED
+ if "Failed" in self.statuses:
+ return EmailDomainSetupStatus.FAILED
+ if "TemporaryFailure" in self.statuses:
+ return EmailDomainSetupStatus.TEMPORARY_FAILURE
+ return EmailDomainSetupStatus.PENDING
+
+ def record_status(self, record: EmailDomainStatusRecord, *, published: bool) -> EmailDomainSetupRecordStatus:
+ if self._is_verified(record.kind, published=published):
+ return EmailDomainSetupRecordStatus.VERIFIED
+ return EmailDomainSetupRecordStatus.FOUND if published else EmailDomainSetupRecordStatus.PENDING
+
+ def _is_verified(self, kind: EmailDomainSetupRecordKind, *, published: bool) -> bool:
+ match kind:
+ case EmailDomainSetupRecordKind.VERIFICATION:
+ return self.verification_status == "Success"
+ case EmailDomainSetupRecordKind.DKIM:
+ return self.dkim_status == "Success"
+ case EmailDomainSetupRecordKind.MAIL_FROM_MX:
+ return self.mail_from_status == "Success"
+ # SES checks only the MX record of the MAIL FROM domain, so its success says nothing about the SPF record.
+ case EmailDomainSetupRecordKind.MAIL_FROM_SPF:
+ return published and self.mail_from_status == "Success"
+ case EmailDomainSetupRecordKind.DMARC:
+ return published
+ case _:
+ return False
+
+
+# Throttling is SES pacing a busy account. The next poll retries it, so reporting it adds only noise.
+def _is_throttling(error: Exception) -> bool:
+ return isinstance(error, ClientError) and error.response["Error"]["Code"] in ("Throttling", "ThrottlingException")
+
+
class SESProvider:
ses_client: "SESClient"
ses_v2_client: "SESV2Client"
ses_v2_metrics_client: "SESV2Client"
ses_v2_recommendations_client: "SESV2Client"
+ ses_status_client: "SESClient"
+ ses_v2_status_client: "SESV2Client"
def __init__(self):
# Initialize the boto3 clients
@@ -289,6 +410,7 @@ class SESProvider:
aws_access_key_id=settings.SES_ACCESS_KEY_ID,
aws_secret_access_key=settings.SES_SECRET_ACCESS_KEY,
region_name=settings.SES_REGION,
+ config=STATUS_READ_CONFIG,
)
self.ses_client = boto3.client(
"ses",
@@ -315,6 +437,20 @@ class SESProvider:
retries={"max_attempts": 1},
),
)
+ self.ses_status_client = boto3.client(
+ "ses",
+ aws_access_key_id=settings.SES_ACCESS_KEY_ID,
+ aws_secret_access_key=settings.SES_SECRET_ACCESS_KEY,
+ region_name=settings.SES_REGION,
+ config=STATUS_READ_CONFIG,
+ )
+ self.ses_v2_status_client = boto3.client(
+ "sesv2",
+ aws_access_key_id=settings.SES_ACCESS_KEY_ID,
+ aws_secret_access_key=settings.SES_SECRET_ACCESS_KEY,
+ region_name=settings.SES_REGION,
+ config=STATUS_READ_CONFIG,
+ )
self.ses_v2_recommendations_client = boto3.client(
"sesv2",
aws_access_key_id=settings.SES_ACCESS_KEY_ID,
@@ -438,9 +574,22 @@ class SESProvider:
return "identity"
return "account"
+ # The cache only saves an STS call, so a cache outage falls back to STS instead of failing SES calls.
@cached_property
def _aws_account_id(self) -> str:
- return self.sts_client.get_caller_identity()["Account"]
+ try:
+ cached: str | None = cache.get(AWS_ACCOUNT_ID_CACHE_KEY)
+ except Exception:
+ logger.warning("Could not read the cached AWS account id", exc_info=True)
+ cached = None
+ if cached is not None:
+ return cached
+ account_id = self.sts_client.get_caller_identity()["Account"]
+ try:
+ cache.set(AWS_ACCOUNT_ID_CACHE_KEY, account_id, AWS_ACCOUNT_ID_CACHE_SECONDS)
+ except Exception:
+ logger.warning("Could not cache the AWS account id", exc_info=True)
+ return account_id
def _identity_arn(self, domain: str) -> str:
return f"arn:aws:ses:{settings.SES_REGION}:{self._aws_account_id}:identity/{domain}"
@@ -455,9 +604,9 @@ class SESProvider:
if e.response["Error"]["Code"] != "AlreadyExistsException":
raise
- def _list_identity_tenants(self, domain: str) -> set[str]:
+ def _list_identity_tenants(self, domain: str, client: "SESV2Client | None" = None) -> set[str]:
try:
- resp = self.ses_v2_client.list_resource_tenants(ResourceArn=self._identity_arn(domain))
+ resp = (client or self.ses_v2_client).list_resource_tenants(ResourceArn=self._identity_arn(domain))
except ClientError as e:
if e.response["Error"]["Code"] == "NotFoundException":
return set()
@@ -532,8 +681,6 @@ class SESProvider:
if not re.match(DOMAIN_REGEX, domain):
raise exceptions.ValidationError("Please enter a valid domain or subdomain name.")
- dns_records: list[EmailDomainDnsRecord] = []
-
# Start/ensure domain verification (TXT at _amazonses.domain) ---
verification_token: str | None = None
try:
@@ -544,17 +691,6 @@ class SESProvider:
if e.response["Error"]["Code"] not in ("InvalidParameterValue",):
raise
- if verification_token:
- dns_records.append(
- {
- "type": "verification",
- "recordType": "TXT",
- "recordHostname": f"_amazonses.{domain}",
- "recordValue": verification_token,
- "status": "pending",
- }
- )
-
# Start/ensure DKIM (three CNAMEs) ---
dkim_tokens: list[str] = []
try:
@@ -564,27 +700,6 @@ class SESProvider:
if e.response["Error"]["Code"] not in ("InvalidParameterValue",):
raise
- for t in dkim_tokens:
- dns_records.append(
- {
- "type": "dkim",
- "recordType": "CNAME",
- "recordHostname": f"{t}._domainkey.{domain}",
- "recordValue": f"{t}.dkim.amazonses.com",
- "status": "pending",
- }
- )
-
- dns_records.append(
- {
- "type": "verification",
- "recordType": "TXT",
- "recordHostname": "@",
- "recordValue": "v=spf1 include:amazonses.com ~all",
- "status": "pending",
- }
- )
-
# Start/ensure MAIL FROM setup (MX + TXT) ---
try:
self.ses_client.set_identity_mail_from_domain(
@@ -596,67 +711,23 @@ class SESProvider:
if e.response["Error"]["Code"] not in ("InvalidParameterValue",):
raise
- ses_region = getattr(settings, "SES_REGION", "us-east-1")
-
- dns_records.append(
- {
- "type": "mail_from",
- "recordType": "MX",
- "recordHostname": f"{mail_from_subdomain}.{domain}",
- "recordValue": f"feedback-smtp.{ses_region}.amazonses.com",
- "priority": 10,
- "status": "pending",
- }
- )
- dns_records.append(
- {
- "type": "mail_from",
- "recordType": "TXT",
- "recordHostname": f"{mail_from_subdomain}.{domain}",
- "recordValue": "v=spf1 include:amazonses.com ~all",
- "status": "pending",
- }
- )
-
- # DMARC — AWS SES has no method to check its presence, so we do a direct DNS
- # lookup further below and include the result in the overall status.
- dns_records.append(
- {
- "type": "dmarc",
- "recordType": "TXT",
- "recordHostname": f"_dmarc.{domain}",
- "recordValue": "v=DMARC1; p=none;",
- "status": "pending",
- }
- )
-
- # Current verification / DKIM statuses to compute overall status & per-record statuses ---
- verification_status: str = "Unknown"
- try:
- id_attrs = self.ses_client.get_identity_verification_attributes(Identities=[domain])
- id_for_domain = id_attrs["VerificationAttributes"].get(domain)
- if id_for_domain is not None:
- verification_status = id_for_domain["VerificationStatus"]
- except ClientError:
- pass
-
- dkim_status: str = "Unknown"
- try:
- dkim_attrs = self.ses_client.get_identity_dkim_attributes(Identities=[domain])
- dkim_for_domain = dkim_attrs["DkimAttributes"].get(domain)
- if dkim_for_domain is not None:
- dkim_status = dkim_for_domain["DkimVerificationStatus"]
- except ClientError:
- pass
+ # DMARC is in the list too: AWS SES has no method to check its presence, so we do a direct
+ # DNS lookup further below and include the result in the overall status.
+ dns_records = [
+ _legacy_dns_record(record)
+ for record in build_email_domain_records(
+ domain=domain,
+ mail_from_subdomain=mail_from_subdomain,
+ verification_token=verification_token,
+ dkim_tokens=dkim_tokens,
+ ses_region=self._ses_region,
+ )
+ ]
- mail_from_status: str = "Unknown"
- try:
- mail_from_attrs = self.ses_client.get_identity_mail_from_domain_attributes(Identities=[domain])
- mail_from_for_domain = mail_from_attrs["MailFromDomainAttributes"].get(domain)
- if mail_from_for_domain is not None:
- mail_from_status = mail_from_for_domain["MailFromDomainStatus"]
- except ClientError:
- pass
+ identity = self._read_identity_state_or_unknown(domain)
+ verification_status = identity.verification_status
+ dkim_status = identity.dkim_status
+ mail_from_status = identity.mail_from_status
# DMARC: check via direct DNS lookup since AWS SES doesn't track it
dmarc_status = "Pending"
@@ -676,26 +747,17 @@ class SESProvider:
except Exception:
logger.exception("Unexpected error during DMARC lookup for %s", domain)
- all_statuses = [verification_status, dkim_status, mail_from_status]
-
# Normalize overall status
overall: EmailDomainVerificationStatus
- if (
- verification_status == "Success"
- and dkim_status == "Success"
- and mail_from_status == "Success"
- and dmarc_status == "Success"
- ):
+ if identity.is_ready(dmarc_published=dmarc_status == "Success"):
overall = "success"
- elif "Failed" in all_statuses:
+ elif "Failed" in identity.statuses:
overall = "failed"
else:
overall = "pending"
- if overall == "success":
- expected_tenant = self._tenant_name_for_team(team_id)
- if expected_tenant not in self._list_identity_tenants(domain):
- overall = "pending"
+ if overall == "success" and not self._is_associated_with_team(domain, team_id):
+ overall = "pending"
# Upgrade per-record statuses if SES reports success
# - Domain verification TXT is considered verified when VerificationStatus == Success
@@ -745,6 +807,76 @@ class SESProvider:
"dnsRecords": dns_records,
}
+ def get_email_domain_status(self, domain: str, mail_from_subdomain: str, team_id: int) -> EmailDomainCheck:
+ try:
+ return self._check_email_domain(domain, mail_from_subdomain=mail_from_subdomain, team_id=team_id)
+ except (ClientError, BotoCoreError) as error:
+ if not _is_throttling(error):
+ capture_exception(error, {"domain": domain, "team_id": team_id})
+ raise EmailProviderUnavailableError(f"Could not read the SES state of {domain}") from error
+
+ def _check_email_domain(self, domain: str, *, mail_from_subdomain: str, team_id: int) -> EmailDomainCheck:
+ identity = self._read_identity_state(domain, self.ses_status_client).for_mail_from_domain(
+ f"{mail_from_subdomain}.{domain}"
+ )
+ if identity.verification_token is None:
+ return build_email_domain_check(provider_status=EmailDomainSetupStatus.NOT_STARTED, records=())
+
+ records = build_email_domain_records(
+ domain=domain,
+ mail_from_subdomain=mail_from_subdomain,
+ verification_token=identity.verification_token,
+ dkim_tokens=identity.dkim_tokens,
+ ses_region=self._ses_region,
+ )
+ published = find_published_records(domain, records)
+ dmarc_published = any(record.kind == EmailDomainSetupRecordKind.DMARC for record in published.records)
+ is_ready = (
+ identity.is_ready(dmarc_published=dmarc_published)
+ and published.every_lookup_answered
+ and self._is_associated_with_team(domain, team_id, self.ses_v2_status_client)
+ )
+ return build_email_domain_check(
+ provider_status=identity.setup_status(is_ready=is_ready),
+ records=[
+ replace(record, status=identity.record_status(record, published=record in published.records))
+ for record in records
+ ],
+ every_dns_lookup_answered=published.every_lookup_answered,
+ )
+
+ @property
+ def _ses_region(self) -> str:
+ return getattr(settings, "SES_REGION", "us-east-1")
+
+ def _is_associated_with_team(self, domain: str, team_id: int, client: "SESV2Client | None" = None) -> bool:
+ return self._tenant_name_for_team(team_id) in self._list_identity_tenants(domain, client)
+
+ def _read_identity_state_or_unknown(self, domain: str) -> "_SESIdentityState":
+ try:
+ return self._read_identity_state(domain)
+ except ClientError:
+ logger.warning("Could not read the SES identity attributes for %s", domain, exc_info=True)
+ return _SESIdentityState.unknown()
+
+ def _read_identity_state(self, domain: str, client: "SESClient | None" = None) -> "_SESIdentityState":
+ ses = client or self.ses_client
+ id_for_domain = ses.get_identity_verification_attributes(Identities=[domain])["VerificationAttributes"].get(
+ domain
+ )
+ dkim_for_domain = ses.get_identity_dkim_attributes(Identities=[domain])["DkimAttributes"].get(domain)
+ mail_from_for_domain = ses.get_identity_mail_from_domain_attributes(Identities=[domain])[
+ "MailFromDomainAttributes"
+ ].get(domain)
+ return _SESIdentityState(
+ verification_status=id_for_domain["VerificationStatus"] if id_for_domain else _UNKNOWN,
+ verification_token=id_for_domain.get("VerificationToken") if id_for_domain else None,
+ dkim_status=dkim_for_domain["DkimVerificationStatus"] if dkim_for_domain else _UNKNOWN,
+ dkim_tokens=tuple(dkim_for_domain.get("DkimTokens", [])) if dkim_for_domain else (),
+ mail_from_status=mail_from_for_domain["MailFromDomainStatus"] if mail_from_for_domain else _UNKNOWN,
+ mail_from_domain=mail_from_for_domain.get("MailFromDomain") if mail_from_for_domain else None,
+ )
+
def update_mail_from_subdomain(self, domain: str, mail_from_subdomain: str):
"""
Update the MAIL FROM subdomain for a given identity
diff --git a/products/workflows/backend/services/email_domain_dns.py b/products/workflows/backend/services/email_domain_dns.py
new file mode 100644
index 000000000000..8d7cb40795ad
--- /dev/null
+++ b/products/workflows/backend/services/email_domain_dns.py
@@ -0,0 +1,155 @@
+import logging
+import ipaddress
+from collections.abc import Sequence
+from concurrent.futures import ThreadPoolExecutor
+
+import dns.rdata
+import dns.resolver
+import dns.exception
+from dns.rdtypes.ANY.CNAME import CNAME
+from dns.rdtypes.ANY.MX import MX
+from dns.rdtypes.ANY.NS import NS
+from dns.rdtypes.ANY.TXT import TXT
+from dns.rdtypes.IN.A import A
+
+from posthog.dataclasses import frozen
+from posthog.security.oob_domains import matched_oob_domain
+
+from products.workflows.backend.facade.contracts import EmailDomainStatusRecord
+from products.workflows.backend.facade.enums import (
+ EmailDomainSetupRecordKind as Kind,
+ EmailDomainSetupRecordType as RecordType,
+)
+
+logger = logging.getLogger(__name__)
+
+LOOKUP_TIMEOUT_SECONDS = 3
+MAX_AUTHORITATIVE_NAMESERVERS = 2
+ABSENT_ERRORS = (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer)
+
+
+@frozen
+class _Lookup:
+ name: str
+ type: RecordType
+
+
+@frozen
+class PublishedRecords:
+ records: frozenset[EmailDomainStatusRecord]
+ unanswered: frozenset[EmailDomainStatusRecord] = frozenset()
+
+ @property
+ def every_lookup_answered(self) -> bool:
+ return not self.unanswered
+
+
+# The zone's own nameservers answer first because public resolvers cache a negative answer for the
+# zone's SOA minimum TTL, which hides a record the customer just added for up to an hour. A public
+# resolver then checks what they could not confirm, because they cannot follow a CNAME to another
+# zone, such as a DMARC record hosted by a DMARC reporting service.
+def find_published_records(domain: str, records: Sequence[EmailDomainStatusRecord]) -> PublishedRecords:
+ if (oob_domain := matched_oob_domain(domain)) is not None:
+ logger.info("security.oob_domain_seen domain=%s guard=email_domain_dns", oob_domain)
+ authoritative = _authoritative_resolver(domain)
+ from_zone = _published_with(authoritative, records) if authoritative else PublishedRecords(records=frozenset())
+ from_public = _published_with(_system_resolver(), [record for record in records if record not in from_zone.records])
+ return PublishedRecords(records=from_zone.records | from_public.records, unanswered=from_public.unanswered)
+
+
+def _published_with(resolver: dns.resolver.Resolver, records: Sequence[EmailDomainStatusRecord]) -> PublishedRecords:
+ if not records:
+ return PublishedRecords(records=frozenset())
+ lookups = {_lookup_for(record) for record in records}
+ with ThreadPoolExecutor(max_workers=len(lookups)) as pool:
+ answers = dict(zip(lookups, pool.map(lambda lookup: _resolve(resolver, lookup), lookups)))
+ return PublishedRecords(
+ records=frozenset(record for record in records if _is_published(record, answers[_lookup_for(record)] or [])),
+ unanswered=frozenset(record for record in records if answers[_lookup_for(record)] is None),
+ )
+
+
+def _lookup_for(record: EmailDomainStatusRecord) -> _Lookup:
+ return _Lookup(name=record.hostname, type=record.type)
+
+
+def _system_resolver() -> dns.resolver.Resolver:
+ resolver = dns.resolver.Resolver()
+ resolver.lifetime = LOOKUP_TIMEOUT_SECONDS
+ return resolver
+
+
+def _authoritative_resolver(domain: str) -> dns.resolver.Resolver | None:
+ system = _system_resolver()
+ try:
+ zone = dns.resolver.zone_for_name(domain, resolver=system, lifetime=LOOKUP_TIMEOUT_SECONDS)
+ nameservers = [rdata.target for rdata in system.resolve(zone, "NS") if isinstance(rdata, NS)]
+ addresses = [
+ rdata.address
+ for nameserver in nameservers[:MAX_AUTHORITATIVE_NAMESERVERS]
+ for rdata in system.resolve(nameserver, "A")
+ if isinstance(rdata, A) and _is_public_address(rdata.address)
+ ]
+ except dns.exception.DNSException:
+ return None
+ if not addresses:
+ return None
+ authoritative = dns.resolver.Resolver(configure=False)
+ authoritative.nameservers = addresses
+ authoritative.lifetime = LOOKUP_TIMEOUT_SECONDS
+ return authoritative
+
+
+# The customer controls the zone's NS records, so they could point our queries at internal addresses.
+def _is_public_address(address: str) -> bool:
+ ip = ipaddress.ip_address(address)
+ return ip.is_global and not ip.is_multicast
+
+
+def _resolve(resolver: dns.resolver.Resolver, lookup: _Lookup) -> list[str] | None:
+ try:
+ answers = resolver.resolve(lookup.name, lookup.type.value)
+ except ABSENT_ERRORS:
+ return []
+ except dns.exception.DNSException:
+ logger.info("No DNS answer for %s %s", lookup.type.value, lookup.name, exc_info=True)
+ return None
+ return [answer for rdata in answers if (answer := _normalized_answer(rdata)) is not None]
+
+
+def _normalized_answer(rdata: dns.rdata.Rdata) -> str | None:
+ if isinstance(rdata, TXT):
+ return b"".join(rdata.strings).decode("utf-8", errors="replace").strip().strip('"')
+ if isinstance(rdata, CNAME):
+ return _normalized_hostname(rdata.target.to_text())
+ if isinstance(rdata, MX):
+ return f"{rdata.preference} {_normalized_hostname(rdata.exchange.to_text())}"
+ return None
+
+
+def _normalized_hostname(hostname: str) -> str:
+ return hostname.lower().rstrip(".")
+
+
+def _is_published(record: EmailDomainStatusRecord, answers: list[str]) -> bool:
+ match record.kind, record.type:
+ case Kind.DMARC, _:
+ return any(answer.lower().startswith("v=dmarc1") for answer in answers)
+ case Kind.SPF | Kind.MAIL_FROM_SPF, _:
+ return any(_spf_includes(answer, record.value) for answer in answers)
+ case _, RecordType.CNAME:
+ return _normalized_hostname(record.value) in answers
+ case _, RecordType.MX:
+ return f"{record.priority} {_normalized_hostname(record.value)}" in answers
+ case _:
+ return record.value in answers
+
+
+# A domain can publish only one SPF record, so SES's mechanism usually sits next to another
+# sender's. The record counts as published when it carries every include the expected one does.
+def _spf_includes(published: str, expected: str) -> bool:
+ published_terms = published.lower().split()
+ if not published_terms or published_terms[0] != "v=spf1":
+ return False
+ expected_includes = {term for term in expected.lower().split() if term.startswith("include:")}
+ return expected_includes.issubset(published_terms)
diff --git a/products/workflows/backend/services/email_domain_status.py b/products/workflows/backend/services/email_domain_status.py
new file mode 100644
index 000000000000..c6a590392036
--- /dev/null
+++ b/products/workflows/backend/services/email_domain_status.py
@@ -0,0 +1,111 @@
+from collections.abc import Sequence
+
+from products.workflows.backend.facade.contracts import EmailDomainCheck, EmailDomainStatusRecord, EmailDomainStatusStep
+from products.workflows.backend.facade.enums import (
+ EmailDomainSetupRecordKind as Kind,
+ EmailDomainSetupRecordStatus as RecordStatus,
+ EmailDomainSetupRecordType as RecordType,
+ EmailDomainSetupStatus as Status,
+ EmailDomainSetupStepKey as StepKey,
+ EmailDomainSetupStepState as StepState,
+)
+
+SES_SPF_VALUE = "v=spf1 include:amazonses.com ~all"
+DEFAULT_DMARC_VALUE = "v=DMARC1; p=none;"
+
+
+def build_email_domain_records(
+ *,
+ domain: str,
+ mail_from_subdomain: str,
+ verification_token: str | None,
+ dkim_tokens: Sequence[str],
+ ses_region: str,
+) -> tuple[EmailDomainStatusRecord, ...]:
+ mail_from_domain = f"{mail_from_subdomain}.{domain}"
+ verification = (
+ [
+ EmailDomainStatusRecord(
+ kind=Kind.VERIFICATION, hostname=f"_amazonses.{domain}", type=RecordType.TXT, value=verification_token
+ )
+ ]
+ if verification_token
+ else []
+ )
+ dkim = [
+ EmailDomainStatusRecord(
+ kind=Kind.DKIM,
+ hostname=f"{token}._domainkey.{domain}",
+ type=RecordType.CNAME,
+ value=f"{token}.dkim.amazonses.com",
+ )
+ for token in dkim_tokens
+ ]
+ return (
+ *verification,
+ *dkim,
+ EmailDomainStatusRecord(kind=Kind.SPF, hostname=domain, type=RecordType.TXT, value=SES_SPF_VALUE),
+ EmailDomainStatusRecord(
+ kind=Kind.MAIL_FROM_MX,
+ hostname=mail_from_domain,
+ type=RecordType.MX,
+ value=f"feedback-smtp.{ses_region}.amazonses.com",
+ priority=10,
+ ),
+ EmailDomainStatusRecord(
+ kind=Kind.MAIL_FROM_SPF, hostname=mail_from_domain, type=RecordType.TXT, value=SES_SPF_VALUE
+ ),
+ EmailDomainStatusRecord(
+ kind=Kind.DMARC, hostname=f"_dmarc.{domain}", type=RecordType.TXT, value=DEFAULT_DMARC_VALUE
+ ),
+ )
+
+
+def build_email_domain_check(
+ *, provider_status: Status, records: Sequence[EmailDomainStatusRecord], every_dns_lookup_answered: bool = True
+) -> EmailDomainCheck:
+ status = _overall_status(provider_status, records)
+ return EmailDomainCheck(
+ status=status,
+ steps=_steps(status, records),
+ records=tuple(records),
+ every_dns_lookup_answered=every_dns_lookup_answered,
+ )
+
+
+def _overall_status(provider_status: Status, records: Sequence[EmailDomainStatusRecord]) -> Status:
+ if provider_status == Status.PENDING and _all_records_published(records):
+ return Status.RECORDS_FOUND
+ return provider_status
+
+
+def _all_records_published(records: Sequence[EmailDomainStatusRecord]) -> bool:
+ return bool(records) and all(record.status != RecordStatus.PENDING for record in records)
+
+
+def _steps(status: Status, records: Sequence[EmailDomainStatusRecord]) -> tuple[EmailDomainStatusStep, ...]:
+ return (
+ EmailDomainStatusStep(key=StepKey.DOMAIN_ADDED, state=_domain_added_state(status)),
+ EmailDomainStatusStep(key=StepKey.RECORDS_FOUND, state=_records_found_state(status, records)),
+ EmailDomainStatusStep(key=StepKey.VERIFIED, state=_verified_state(status)),
+ )
+
+
+def _domain_added_state(status: Status) -> StepState:
+ return StepState.PENDING if status == Status.NOT_STARTED else StepState.DONE
+
+
+def _records_found_state(status: Status, records: Sequence[EmailDomainStatusRecord]) -> StepState:
+ if status == Status.VERIFIED or _all_records_published(records):
+ return StepState.DONE
+ if status in (Status.FAILED, Status.TEMPORARY_FAILURE):
+ return StepState.FAILED
+ return StepState.PENDING
+
+
+def _verified_state(status: Status) -> StepState:
+ if status == Status.VERIFIED:
+ return StepState.DONE
+ if status == Status.FAILED:
+ return StepState.FAILED
+ return StepState.PENDING
diff --git a/products/workflows/backend/test/email_domain_api_test_case.py b/products/workflows/backend/test/email_domain_api_test_case.py
new file mode 100644
index 000000000000..fb1ed8e1ae75
--- /dev/null
+++ b/products/workflows/backend/test/email_domain_api_test_case.py
@@ -0,0 +1,108 @@
+from typing import Any
+
+from posthog.test.base import APIBaseTest
+from unittest.mock import patch
+
+from django.core.cache import cache
+from django.test import override_settings
+
+from rest_framework import status
+
+from posthog.models.integration import Integration
+from posthog.models.organization import OrganizationMembership
+from posthog.models.personal_api_key import PersonalAPIKey
+from posthog.models.team import Team
+from posthog.models.utils import hash_key_value
+
+from products.workflows.backend.tests.fakes.world import EmailDomainWorld
+
+DOMAIN = "mail.example.com"
+ROOT_DOMAIN = "example.com"
+ZONE_NAMESERVERS = {"ns1.example-dns.net": "34.120.0.1", "ns2.example-dns.net": "34.120.0.2"}
+ALL_KINDS = ("verification", "dkim", "spf", "mail_from_mx", "mail_from_spf", "dmarc")
+
+
+@override_settings(SES_REGION="us-east-1", SES_TENANT_CONFIGURATION_SETS=[])
+class EmailDomainApiTestCase(APIBaseTest):
+ zone_nameservers: dict[str, str] = ZONE_NAMESERVERS
+
+ def setUp(self) -> None:
+ super().setUp()
+ cache.clear()
+ self.organization_membership.level = OrganizationMembership.Level.ADMIN
+ self.organization_membership.save()
+ # Polls and verifies run as an admin's token, the way the setup agent calls them.
+ PersonalAPIKey.objects.create(
+ label="Admin", user=self.user, secure_value=hash_key_value("admin_key"), scopes=["integration:write"]
+ )
+ self.world = self.enterContext(EmailDomainWorld().installed())
+ self.world.dns.add_zone(ROOT_DOMAIN, nameservers=self.zone_nameservers)
+ self.reload_workers = self.enterContext(
+ patch("posthog.models.integration.email.reload_integrations_on_workers")
+ )
+ self.report_user_action = self.enterContext(patch("posthog.api.integration.report_user_action"))
+ self.feature_flags = self.enterContext(
+ patch(
+ "posthoganalytics.feature_enabled",
+ side_effect=lambda key, *args, **kwargs: key == "workflows-email-domain-wizard",
+ )
+ )
+
+ def create_sender(self, email: str, *, team: Team | None = None) -> int:
+ response = self.client.post(
+ f"/api/projects/{(team or self.team).id}/integrations",
+ {"kind": "email", "config": {"email": email, "name": "Example", "provider": "ses"}},
+ format="json",
+ )
+ assert response.status_code == status.HTTP_201_CREATED, response.json()
+ return response.json()["id"]
+
+ def get_status(self, integration_id: int, *, refresh: bool = True, as_browser: bool = False) -> Any:
+ url = f"/api/projects/{self.team.id}/integrations/{integration_id}/email/status"
+ params = {"refresh": str(refresh).lower()}
+ if as_browser:
+ return self.client.get(url, params)
+ return self.client.get(url, params, HTTP_AUTHORIZATION="Bearer admin_key")
+
+ def verify(self, integration_id: int, *, key: str = "admin_key") -> Any:
+ return self.client.post(
+ f"/api/projects/{self.team.id}/integrations/{integration_id}/email/verify",
+ HTTP_AUTHORIZATION=f"Bearer {key}",
+ )
+
+ def status_body(self, integration_id: int, *, refresh: bool = True) -> dict[str, Any]:
+ response = self.get_status(integration_id, refresh=refresh)
+ assert response.status_code == status.HTTP_200_OK, response.json()
+ return response.json()
+
+ def records_to_publish(self, integration_id: int, *, domain: str = DOMAIN) -> list[dict[str, Any]]:
+ identity = self.world.ses.identities[domain]
+ identity.checks_paused = True
+ records = self.status_body(integration_id)["records"]
+ identity.checks_paused = False
+ return records
+
+ def publish(self, records: list[dict[str, Any]], *kinds: str) -> None:
+ for record in records:
+ if record["kind"] in kinds:
+ self.world.publish_record(record)
+
+ def is_verified(self, integration_id: int) -> bool:
+ return Integration.objects.get(id=integration_id).config["verified"]
+
+ def verified_events(self) -> list[Any]:
+ return [call for call in self.report_user_action.call_args_list if call.args[1] == "email domain verified"]
+
+ def verified_event_methods(self) -> list[str]:
+ return [call.args[2]["method"] for call in self.verified_events()]
+
+
+def record_statuses(body: dict[str, Any]) -> dict[str, list[str]]:
+ statuses: dict[str, list[str]] = {}
+ for record in body["records"]:
+ statuses.setdefault(record["kind"], []).append(record["status"])
+ return statuses
+
+
+def step_states(body: dict[str, Any]) -> list[str]:
+ return [step["state"] for step in body["steps"]]
diff --git a/products/workflows/backend/test/test_email_domain_status.py b/products/workflows/backend/test/test_email_domain_status.py
new file mode 100644
index 000000000000..2f674290ebca
--- /dev/null
+++ b/products/workflows/backend/test/test_email_domain_status.py
@@ -0,0 +1,54 @@
+from django.test import SimpleTestCase
+
+from parameterized import parameterized
+
+from products.workflows.backend.facade.contracts import EmailDomainStatusRecord
+from products.workflows.backend.facade.enums import (
+ EmailDomainSetupRecordKind as Kind,
+ EmailDomainSetupRecordType as RecordType,
+)
+from products.workflows.backend.services.email_domain_dns import find_published_records
+from products.workflows.backend.tests.fakes.world import EmailDomainWorld
+
+DOMAIN = "mail.example.com"
+
+
+def _record(kind: Kind, record_type: RecordType, hostname: str, value: str, priority: int | None = None):
+ return EmailDomainStatusRecord(kind=kind, hostname=hostname, type=record_type, value=value, priority=priority)
+
+
+SPF = _record(Kind.SPF, RecordType.TXT, DOMAIN, "v=spf1 include:amazonses.com ~all")
+VERIFICATION = _record(Kind.VERIFICATION, RecordType.TXT, f"_amazonses.{DOMAIN}", "tok")
+DKIM = _record(Kind.DKIM, RecordType.CNAME, f"dk1._domainkey.{DOMAIN}", "dk1.dkim.amazonses.com")
+MAIL_FROM_MX = _record(
+ Kind.MAIL_FROM_MX, RecordType.MX, f"feedback.{DOMAIN}", "feedback-smtp.us-east-1.amazonses.com", priority=10
+)
+DMARC = _record(Kind.DMARC, RecordType.TXT, f"_dmarc.{DOMAIN}", "v=DMARC1; p=none;")
+
+
+class TestFindPublishedRecords(SimpleTestCase):
+ @parameterized.expand(
+ [
+ ("txt_split_into_chunks", VERIFICATION, '"to" "k"', True),
+ ("txt_other_value", VERIFICATION, '"other"', False),
+ (
+ "spf_merged_with_another_sender",
+ SPF,
+ '"v=spf1 include:_spf.google.com include:amazonses.com -all"',
+ True,
+ ),
+ ("spf_without_ses_include", SPF, '"v=spf1 include:_spf.google.com ~all"', False),
+ ("cname_in_upper_case", DKIM, "DK1.DKIM.AMAZONSES.COM.", True),
+ ("mx_with_other_priority", MAIL_FROM_MX, "20 feedback-smtp.us-east-1.amazonses.com.", False),
+ ("existing_strict_dmarc", DMARC, '"v=DMARC1; p=reject; rua=mailto:[email protected]"', True),
+ ]
+ )
+ def test_matches_published_value(
+ self, _name: str, record: EmailDomainStatusRecord, published: str, expected: bool
+ ) -> None:
+ world = EmailDomainWorld()
+ world.dns.add_zone("example.com", nameservers={"ns1.example-dns.net": "34.120.0.1"})
+ world.dns.publish(record.hostname, record.type.value, published)
+
+ with world.installed():
+ assert (record in find_published_records(DOMAIN, [record]).records) is expected
diff --git a/products/workflows/backend/test/test_email_domain_status_api.py b/products/workflows/backend/test/test_email_domain_status_api.py
new file mode 100644
index 000000000000..add5cce7168f
--- /dev/null
+++ b/products/workflows/backend/test/test_email_domain_status_api.py
@@ -0,0 +1,436 @@
+from collections.abc import Callable
+from itertools import product
+
+from unittest.mock import ANY, patch
+
+from django.core.cache import cache
+
+from parameterized import parameterized
+from rest_framework import status
+
+from posthog.models.integration import Integration
+from posthog.models.integration.email import EmailIntegration
+from posthog.models.organization import Organization
+from posthog.models.personal_api_key import PersonalAPIKey
+from posthog.models.team import Team
+from posthog.models.utils import hash_key_value
+
+from products.workflows.backend.models.team_workflows_config import TeamWorkflowsConfig
+from products.workflows.backend.test.email_domain_api_test_case import (
+ ALL_KINDS,
+ DOMAIN,
+ ROOT_DOMAIN,
+ EmailDomainApiTestCase,
+ record_statuses,
+ step_states,
+)
+from products.workflows.backend.tests.fakes.fake_dns import dns_operation
+from products.workflows.backend.tests.fakes.faults import AwsFault, DnsFault
+from products.workflows.backend.tests.fakes.world import EmailDomainWorld
+
+SES_STATUS_READS = (
+ "ses.get_identity_verification_attributes",
+ "ses.get_identity_dkim_attributes",
+ "ses.get_identity_mail_from_domain_attributes",
+ "sts.get_caller_identity",
+ "sesv2.list_resource_tenants",
+)
+
+
+class TestEmailDomainVerification(EmailDomainApiTestCase):
+ def test_senders_become_verified_as_the_records_are_published_and_the_event_fires_once(self) -> None:
+ sender = self.create_sender(f"hello@{DOMAIN}")
+ sibling = self.create_sender(f"news@{DOMAIN}")
+ self.world.ses.identities[DOMAIN].checks_paused = True
+
+ body = self.status_body(sender)
+ assert (body["status"], body["verified"], step_states(body)) == (
+ "pending",
+ False,
+ ["done", "pending", "pending"],
+ )
+ assert {status for statuses in record_statuses(body).values() for status in statuses} == {"pending"}
+ records = body["records"]
+
+ for kind in ALL_KINDS[:-1]:
+ self.publish(records, kind)
+ body = self.status_body(sender)
+ assert (body["status"], set(record_statuses(body)[kind])) == ("pending", {"found"}), kind
+
+ self.publish(records, "dmarc")
+ body = self.status_body(sender)
+ assert (body["status"], step_states(body)) == ("records_found", ["done", "done", "pending"])
+ assert not self.is_verified(sender)
+ assert self.verified_events() == []
+
+ self.world.ses.identities[DOMAIN].checks_paused = False
+ body = self.status_body(sender)
+ assert (body["status"], body["verified"], step_states(body)) == ("verified", False, ["done", "done", "done"])
+ assert record_statuses(body) == {
+ "verification": ["verified"],
+ "dkim": ["verified"] * 3,
+ "spf": ["found"],
+ "mail_from_mx": ["verified"],
+ "mail_from_spf": ["verified"],
+ "dmarc": ["verified"],
+ }
+ assert not self.is_verified(sender)
+
+ assert self.verify(sender).status_code == status.HTTP_200_OK
+ assert self.status_body(sender)["verified"] is True
+ assert self.is_verified(sender) and self.is_verified(sibling)
+ assert TeamWorkflowsConfig.objects.get(team_id=self.team.id).default_email_integration_id == sender
+ assert self.verified_events() == [
+ ((ANY, "email domain verified", {"method": "agent", "seconds_since_created": ANY}), {"team": ANY})
+ ]
+ self.reload_workers.assert_called_once_with(self.team.id, sorted([sender, sibling]))
+ assert self.world.ses.writes.count("verify_domain_identity") == 3
+
+ late_sender = self.create_sender(f"billing@{DOMAIN}")
+ assert self.verify(late_sender).status_code == status.HTTP_200_OK
+ assert self.is_verified(late_sender)
+ assert len(self.verified_events()) == 1
+
+ @parameterized.expand(
+ [(False, False, 404), (True, False, 200), (False, True, 200), (True, True, 200), (None, None, 404)]
+ )
+ def test_status_reads_ses_without_writing_to_it(
+ self, wizard_enabled: bool | None, agent_enabled: bool | None, expected_status: int
+ ) -> None:
+ sender = self.create_sender(f"hello@{DOMAIN}")
+ self.publish(self.records_to_publish(sender), *ALL_KINDS)
+ writes_after_create = list(self.world.ses.writes)
+ self.feature_flags.side_effect = lambda key, *args, **kwargs: {
+ "workflows-email-domain-wizard": wizard_enabled,
+ "workflows-email-domain-agent-setup": agent_enabled,
+ }.get(key, False)
+ response = self.get_status(sender)
+ assert response.status_code == expected_status, response.json()
+ if expected_status == 200:
+ assert response.json()["status"] == "verified"
+
+ assert self.world.ses.writes == writes_after_create
+
+ def test_status_is_cached_until_refresh(self) -> None:
+ sender = self.create_sender(f"hello@{DOMAIN}")
+ records = self.records_to_publish(sender)
+ assert self.status_body(sender, refresh=False)["status"] == "pending"
+
+ self.publish(records, *ALL_KINDS)
+
+ assert self.status_body(sender, refresh=False)["status"] == "pending"
+ assert self.status_body(sender, refresh=True)["status"] == "verified"
+
+ @parameterized.expand(
+ [
+ ("write_scope_records_it", ["integration:write"], status.HTTP_200_OK, True, ["agent"]),
+ ("read_scope_cannot_record_it", ["integration:read"], status.HTTP_403_FORBIDDEN, False, []),
+ ]
+ )
+ def test_only_email_verify_records_the_verification_for_a_personal_api_key(
+ self, _name: str, scopes: list[str], verify_status: int, verified: bool, event_methods: list[str]
+ ) -> None:
+ sender = self.create_sender(f"hello@{DOMAIN}")
+ self.publish(self.records_to_publish(sender), *ALL_KINDS)
+ PersonalAPIKey.objects.create(
+ label="Agent", user=self.user, secure_value=hash_key_value("agent_key"), scopes=scopes
+ )
+ self.client.logout()
+ self.client.credentials(HTTP_AUTHORIZATION="Bearer agent_key")
+
+ body = self.client.get(f"/api/projects/{self.team.id}/integrations/{sender}/email/status?refresh=true").json()
+ assert (body["status"], body["verified"], self.is_verified(sender)) == ("verified", False, False)
+
+ assert self.verify(sender, key="agent_key").status_code == verify_status
+ assert self.is_verified(sender) is verified
+ assert self.verified_event_methods() == event_methods
+
+ def test_a_cache_outage_still_answers_from_a_live_check(self) -> None:
+ sender = self.create_sender(f"hello@{DOMAIN}")
+ self.publish(self.records_to_publish(sender), *ALL_KINDS)
+
+ with (
+ patch("posthog.models.integration.email.cache.get", side_effect=ConnectionError("Redis is down")),
+ patch("posthog.models.integration.email.cache.set", side_effect=ConnectionError("Redis is down")),
+ ):
+ report = EmailIntegration(Integration.objects.get(id=sender)).status(refresh=False)
+
+ assert (report.check.status, report.verified) == ("verified", False)
+
+ def test_a_browser_session_poll_reports_but_only_email_verify_records_the_verification(self) -> None:
+ sender = self.create_sender(f"hello@{DOMAIN}")
+ self.publish(self.records_to_publish(sender), *ALL_KINDS)
+
+ response = self.get_status(sender, as_browser=True)
+
+ assert response.status_code == status.HTTP_200_OK
+ assert (response.json()["status"], response.json()["verified"]) == ("verified", False)
+ assert (self.is_verified(sender), self.verified_events()) == (False, [])
+
+ verify = self.client.post(f"/api/projects/{self.team.id}/integrations/{sender}/email/verify")
+
+ assert verify.status_code == status.HTTP_200_OK
+ assert self.is_verified(sender)
+ assert self.verified_event_methods() == ["manual"]
+
+ def test_failed_worker_reload_keeps_the_verified_answer_and_is_not_retried(self) -> None:
+ sender = self.create_sender(f"hello@{DOMAIN}")
+ self.publish(self.records_to_publish(sender), *ALL_KINDS)
+ self.reload_workers.side_effect = ConnectionError("Redis is down")
+
+ with patch("posthog.models.integration.email.capture_exception") as capture_exception:
+ assert self.verify(sender).status_code == status.HTTP_200_OK
+ capture_exception.assert_called_once()
+
+ self.reload_workers.side_effect = None
+ assert self.status_body(sender)["verified"] is True
+ assert self.is_verified(sender)
+ assert self.reload_workers.call_count == 1
+ assert len(self.verified_events()) == 1
+
+
+class TestEmailDomainPartialSesState(EmailDomainApiTestCase):
+ @parameterized.expand(
+ [
+ (
+ "verification_token_missing",
+ lambda world: setattr(world.ses.identities[DOMAIN], "verification_token", None),
+ "not_started",
+ {},
+ ),
+ (
+ "dkim_never_enabled",
+ lambda world: setattr(world.ses.identities[DOMAIN], "dkim_tokens", ()),
+ "records_found",
+ {
+ "verification": ["verified"],
+ "spf": ["found"],
+ "mail_from_mx": ["verified"],
+ "mail_from_spf": ["verified"],
+ "dmarc": ["verified"],
+ },
+ ),
+ (
+ "only_two_of_three_dkim_records_published",
+ lambda world: world.dns.unpublish(
+ f"{world.ses.identities[DOMAIN].dkim_tokens[2]}._domainkey.{DOMAIN}", "CNAME"
+ ),
+ "pending",
+ {"dkim": ["found", "found", "pending"]},
+ ),
+ (
+ "mail_from_never_set",
+ lambda world: setattr(world.ses.identities[DOMAIN], "mail_from_domain", None),
+ "records_found",
+ {"mail_from_mx": ["found"], "mail_from_spf": ["found"]},
+ ),
+ (
+ "mail_from_set_for_another_senders_subdomain",
+ lambda world: setattr(world.ses.identities[DOMAIN], "mail_from_domain", f"bounce.{DOMAIN}"),
+ "records_found",
+ {"mail_from_mx": ["found"], "mail_from_spf": ["found"]},
+ ),
+ (
+ "mail_from_mx_published_without_its_spf_record",
+ lambda world: world.dns.unpublish(f"feedback.{DOMAIN}", "TXT"),
+ "verified",
+ {"mail_from_mx": ["verified"], "mail_from_spf": ["pending"]},
+ ),
+ (
+ "second_mx_record_on_the_bounce_subdomain",
+ lambda world: world.dns.publish(f"feedback.{DOMAIN}", "MX", "20 mx.example.net."),
+ "records_found",
+ {"mail_from_mx": ["found"], "mail_from_spf": ["found"]},
+ ),
+ (
+ "identity_owned_by_another_teams_tenant",
+ lambda world: world.ses.transfer_identity(DOMAIN, "team-999999"),
+ "records_found",
+ {"verification": ["verified"], "dkim": ["verified"] * 3},
+ ),
+ (
+ "dkim_failed",
+ lambda world: setattr(world.ses.identities[DOMAIN], "forced_dkim_status", "Failed"),
+ "failed",
+ {"dkim": ["found"] * 3},
+ ),
+ (
+ "verification_temporary_failure",
+ lambda world: setattr(world.ses.identities[DOMAIN], "forced_verification_status", "TemporaryFailure"),
+ "temporary_failure",
+ {"verification": ["found"]},
+ ),
+ ]
+ )
+ def test_reports_partial_ses_state_without_verifying(
+ self,
+ _name: str,
+ arrange: Callable[[EmailDomainWorld], object],
+ expected_status: str,
+ expected_records: dict[str, list[str]],
+ ) -> None:
+ sender = self.create_sender(f"hello@{DOMAIN}")
+ self.publish(self.records_to_publish(sender), *ALL_KINDS)
+ arrange(self.world)
+
+ body = self.status_body(sender)
+
+ assert body["status"] == expected_status
+ assert {kind: statuses for kind, statuses in record_statuses(body).items() if kind in expected_records} == (
+ expected_records
+ )
+ if expected_status != "verified":
+ assert (body["verified"], self.is_verified(sender), self.verified_events()) == (False, False, [])
+
+
+class TestEmailDomainTenantScoping(EmailDomainApiTestCase):
+ def test_verifying_a_domain_leaves_other_teams_senders_on_it_untouched(self) -> None:
+ other_team = Team.objects.create(organization=self.organization, name="Other project")
+ sender = self.create_sender(f"hello@{DOMAIN}")
+ other_teams_sender = self.create_sender(f"hello@{DOMAIN}", team=other_team)
+ self.publish(self.records_to_publish(sender), *ALL_KINDS)
+
+ assert self.verify(sender).status_code == status.HTTP_200_OK
+
+ assert self.is_verified(sender)
+ assert not self.is_verified(other_teams_sender)
+ self.reload_workers.assert_called_once_with(self.team.id, [sender])
+
+ def test_a_sender_of_another_organization_is_not_found(self) -> None:
+ other_team = Team.objects.create(organization=Organization.objects.create(name="Other"), name="Other")
+ foreign_sender = Integration.objects.create(
+ team=other_team,
+ kind="email",
+ integration_id=f"hello@{DOMAIN}",
+ config={"email": f"hello@{DOMAIN}", "domain": DOMAIN, "provider": "ses", "verified": False},
+ )
+
+ response = self.get_status(foreign_sender.id)
+
+ assert response.status_code == status.HTTP_404_NOT_FOUND
+ assert not self.is_verified(foreign_sender.id)
+
+
+class TestEmailDomainDnsLookups(EmailDomainApiTestCase):
+ def test_dmarc_record_hosted_in_another_zone_through_a_cname_counts(self) -> None:
+ self.world.dns.add_zone("dmarc-reports.example.net")
+ sender = self.create_sender(f"hello@{DOMAIN}")
+ self.publish(self.records_to_publish(sender), *ALL_KINDS[:-1])
+ self.world.dns.publish(f"_dmarc.{DOMAIN}", "CNAME", f"{DOMAIN}._dmarc.dmarc-reports.example.net.")
+ self.world.dns.publish_txt(f"{DOMAIN}._dmarc.dmarc-reports.example.net", "v=DMARC1; p=reject")
+
+ body = self.status_body(sender)
+
+ assert (body["status"], record_statuses(body)["dmarc"]) == ("verified", ["verified"])
+
+ @parameterized.expand([(DnsFault.TIMEOUT,), (DnsFault.SERVFAIL,)])
+ def test_public_resolver_confirms_a_record_the_zone_nameservers_cannot_answer(self, fault: DnsFault) -> None:
+ sender = self.create_sender(f"hello@{DOMAIN}")
+ self.publish(self.records_to_publish(sender), *ALL_KINDS)
+ self.world.faults.inject(dns_operation(f"_dmarc.{DOMAIN}", "TXT", at="zone"), fault)
+
+ assert self.status_body(sender)["status"] == "verified"
+
+ def test_zone_nameservers_confirm_a_record_the_public_resolver_has_not_seen_yet(self) -> None:
+ sender = self.create_sender(f"hello@{DOMAIN}")
+ self.publish(self.records_to_publish(sender), *ALL_KINDS)
+ self.world.faults.inject(dns_operation(f"_dmarc.{DOMAIN}", "TXT", at="public"), DnsFault.NXDOMAIN)
+
+ assert self.status_body(sender)["status"] == "verified"
+
+ @parameterized.expand(
+ [
+ ("zone_apex_lookup", ROOT_DOMAIN, "SOA"),
+ ("nameserver_lookup", ROOT_DOMAIN, "NS"),
+ ("nameserver_address_lookup", "ns1.example-dns.net", "A"),
+ ]
+ )
+ def test_failing_zone_nameserver_discovery_falls_back_to_the_public_resolver(
+ self, _name: str, name: str, rdtype: str
+ ) -> None:
+ sender = self.create_sender(f"hello@{DOMAIN}")
+ self.publish(self.records_to_publish(sender), *ALL_KINDS)
+ self.world.faults.inject(dns_operation(name, rdtype), DnsFault.SERVFAIL)
+
+ assert self.status_body(sender)["status"] == "verified"
+
+ def test_txt_records_split_into_several_strings_count(self) -> None:
+ sender = self.create_sender(f"hello@{DOMAIN}")
+ records = self.records_to_publish(sender)
+ self.publish(records, "verification", "dkim", "mail_from_mx")
+ self.world.dns.publish(DOMAIN, "TXT", '"v=spf1 include:" "amazonses.com ~all"')
+ self.world.dns.publish(f"feedback.{DOMAIN}", "TXT", '"v=spf1 " "include:amazonses.com ~all"')
+ self.world.dns.publish(f"_dmarc.{DOMAIN}", "TXT", '"v=DMARC1; " "p=none;"')
+
+ body = self.status_body(sender)
+
+ assert body["status"] == "verified"
+ assert record_statuses(body)["spf"] == ["found"]
+
+ def test_never_queries_a_zone_nameserver_at_an_internal_address(self) -> None:
+ self.world.dns.add_zone("internal.example.org", nameservers={"ns.internal.example.org": "10.0.0.53"})
+ sender = self.create_sender("[email protected]")
+ self.publish(self.records_to_publish(sender, domain="internal.example.org"), *ALL_KINDS)
+
+ assert self.status_body(sender)["status"] == "verified"
+ assert "10.0.0.53" not in self.world.dns.queried_addresses
+
+
+class TestEmailDomainStatusChaos(EmailDomainApiTestCase):
+ def setUp(self) -> None:
+ super().setUp()
+ self.capture_exception = self.enterContext(patch("products.workflows.backend.providers.ses.capture_exception"))
+ self.sender = self.create_sender(f"hello@{DOMAIN}")
+ self.publish(self.records_to_publish(self.sender), *ALL_KINDS)
+ cache.clear()
+
+ @parameterized.expand(
+ product(SES_STATUS_READS, list(AwsFault)), name_func=lambda f, _n, p: f"{f.__name__}_{p.args[0]}_{p.args[1]}"
+ )
+ def test_aws_failure_answers_503_and_the_next_poll_recovers(self, operation: str, fault: AwsFault) -> None:
+ self.world.faults.inject(operation, fault)
+
+ response = self.get_status(self.sender, refresh=False)
+
+ assert response.status_code == status.HTTP_503_SERVICE_UNAVAILABLE
+ assert response.json() == {
+ "type": "server_error",
+ "code": "email_provider_unavailable",
+ "detail": "Couldn't reach the email provider to check this domain. Try again in a moment.",
+ "attr": None,
+ }
+ assert (self.is_verified(self.sender), self.verified_events()) == (False, [])
+ assert self.capture_exception.called is (fault != AwsFault.THROTTLING)
+
+ self.world.faults.clear()
+ assert self.status_body(self.sender, refresh=False)["status"] == "verified"
+
+ @parameterized.expand(
+ product(
+ [
+ ("verification", f"_amazonses.{DOMAIN}", "TXT", "verified"),
+ ("dkim", None, "CNAME", "verified"),
+ ("spf", DOMAIN, "TXT", "found"),
+ ("mail_from_mx", f"feedback.{DOMAIN}", "MX", "verified"),
+ ("mail_from_spf", f"feedback.{DOMAIN}", "TXT", "verified"),
+ ("dmarc", f"_dmarc.{DOMAIN}", "TXT", "verified"),
+ ],
+ [DnsFault.TIMEOUT, DnsFault.SERVFAIL],
+ ),
+ name_func=lambda f, _n, p: f"{f.__name__}_{p.args[0][0]}_{p.args[1]}",
+ )
+ def test_unanswered_dns_lookup_holds_verification_until_it_answers(
+ self, lookup: tuple[str, str | None, str, str], fault: DnsFault
+ ) -> None:
+ kind, hostname, rdtype, status_once_answered = lookup
+ dkim_token = self.world.ses.identities[DOMAIN].dkim_tokens[0]
+ self.world.faults.inject(dns_operation(hostname or f"{dkim_token}._domainkey.{DOMAIN}", rdtype), fault)
+
+ body = self.status_body(self.sender, refresh=False)
+
+ assert body["status"] in ("pending", "records_found")
+ assert (body["verified"], self.is_verified(self.sender), self.verified_events()) == (False, False, [])
+
+ self.world.faults.clear()
+ body = self.status_body(self.sender, refresh=False)
+ assert (body["status"], record_statuses(body)[kind][0]) == ("verified", status_once_answered)
diff --git a/products/workflows/backend/test/test_email_integration_ses.py b/products/workflows/backend/test/test_email_integration_ses.py
index ad8f8c6380c4..11d2c9cb1a3b 100644
--- a/products/workflows/backend/test/test_email_integration_ses.py
+++ b/products/workflows/backend/test/test_email_integration_ses.py
@@ -2,12 +2,15 @@ import pytest
from posthog.test.base import BaseTest
from unittest.mock import MagicMock, patch
+from django.core.cache import cache
+
from posthog.api.integration import IntegrationViewSet
from posthog.models.integration import EmailIntegration, Integration
from posthog.models.organization import Organization
from posthog.models.team.team import Team
from products.workflows.backend.providers import SESProvider
+from products.workflows.backend.providers.ses import AWS_ACCOUNT_ID_CACHE_KEY
class TestEmailIntegrationCrossTenantStaleVerification(BaseTest):
@@ -159,12 +162,14 @@ class TestEmailIntegrationCrossTenantStaleVerification(BaseTest):
integration_b.refresh_from_db()
assert integration_b.config.get("verified") is False
- def test_aws_account_id_is_cached_per_provider(self):
- provider = self._build_ses_provider()
- provider.sts_client.get_caller_identity.reset_mock()
+ def test_aws_account_id_is_looked_up_once_across_providers(self):
+ cache.delete(AWS_ACCOUNT_ID_CACHE_KEY)
+ providers = [self._build_ses_provider(), self._build_ses_provider()]
+ for provider in providers:
+ provider.sts_client.get_caller_identity.reset_mock()
- for _ in range(5):
+ for provider in providers:
provider._identity_arn("partner.com")
provider._identity_arn("other.com")
- assert provider.sts_client.get_caller_identity.call_count == 1
+ assert sum(provider.sts_client.get_caller_identity.call_count for provider in providers) == 1
diff --git a/products/workflows/backend/test/test_ses_provider.py b/products/workflows/backend/test/test_ses_provider.py
index 6eab44e15ddf..27b63586f39a 100644
--- a/products/workflows/backend/test/test_ses_provider.py
+++ b/products/workflows/backend/test/test_ses_provider.py
@@ -132,6 +132,7 @@ class TestSESProvider(TestCase):
@patch("products.workflows.backend.providers.ses.boto3.client")
def test_create_email_domain_invalid_domain(self, mock_boto_client):
+ mock_boto_client.return_value.get_caller_identity.return_value = {"Account": "123456789012"}
with override_settings(
SES_ACCESS_KEY_ID="test_access_key", SES_SECRET_ACCESS_KEY="test_secret_key", SES_REGION="us-east-1"
):
diff --git a/products/workflows/backend/tests/fakes/fake_dns.py b/products/workflows/backend/tests/fakes/fake_dns.py
new file mode 100644
index 000000000000..bb371b834dbb
--- /dev/null
+++ b/products/workflows/backend/tests/fakes/fake_dns.py
@@ -0,0 +1,214 @@
+import time
+import threading
+from collections.abc import Iterator
+from contextlib import ExitStack, contextmanager
+from typing import Any, Literal
+
+from unittest.mock import patch
+
+import dns.name
+import dns.query
+import dns.rcode
+import dns.rdata
+import dns.rrset
+import dns.message
+import dns.resolver
+import dns.exception
+import dns.rdatatype
+import dns.rdataclass
+from dns.rdtypes.ANY.CNAME import CNAME
+from dns.rdtypes.ANY.TXT import TXT
+
+from posthog.dataclasses import frozen
+
+from products.workflows.backend.tests.fakes.faults import DnsFault, FaultInjector
+
+PUBLIC_RESOLVER_ADDRESS = "8.8.8.8"
+UNANSWERED_QUERY_SECONDS = 2.0
+MAX_CNAME_CHAIN = 8
+TTL = 300
+
+RecordKey = tuple[dns.name.Name, dns.rdatatype.RdataType]
+Lookup = Literal["zone", "public"]
+
+
+def dns_operation(name: str, rdtype: str, *, at: Lookup | None = None) -> str:
+ """Fault key for one lookup. `at` narrows it to the zone's own nameservers or to the public resolver."""
+ lookup = f"{_relative_text(_absolute(name))}/{rdtype.upper()}"
+ return f"dns:{at}:{lookup}" if at else f"dns:{lookup}"
+
+
+@frozen
+class _Zone:
+ apex: dns.name.Name
+ nameserver_addresses: tuple[str, ...]
+
+
+@frozen
+class _Resolution:
+ rcode: dns.rcode.Rcode
+ answer: tuple[dns.rrset.RRset, ...] = ()
+ authority: tuple[dns.rrset.RRset, ...] = ()
+
+
+class _ResolverClock:
+ """Simulated time for dnspython, so an unanswered query uses up the resolver's lifetime without a real wait.
+ Each thread keeps its own offset because the code under test runs lookups in parallel threads."""
+
+ def __init__(self) -> None:
+ self._offsets = threading.local()
+
+ def time(self) -> float:
+ return time.time() + self._offset()
+
+ def sleep(self, seconds: float) -> None:
+ self._offsets.value = self._offset() + seconds
+
+ def __getattr__(self, name: str) -> Any:
+ return getattr(time, name)
+
+ def _offset(self) -> float:
+ return getattr(self._offsets, "value", 0.0)
+
+
+class FakeDnsZone:
+ """The DNS seen through dnspython: zones with their own nameservers, and a public resolver that follows
+ CNAME chains across zones. It answers at the network edge (`dns.query`), so the real `Resolver`,
+ `zone_for_name` and answer parsing run unchanged."""
+
+ def __init__(self, faults: FaultInjector) -> None:
+ self._faults = faults
+ self._zones: list[_Zone] = []
+ self._records: dict[RecordKey, list[dns.rdata.Rdata]] = {}
+ self._clock = _ResolverClock()
+ self.queried_addresses: set[str] = set()
+
+ @contextmanager
+ def installed(self) -> Iterator["FakeDnsZone"]:
+ with ExitStack() as stack:
+ stack.enter_context(patch("dns.query.udp", self._answer_query))
+ stack.enter_context(patch("dns.query.tcp", self._answer_query))
+ stack.enter_context(patch.object(dns.resolver.Resolver, "read_resolv_conf", _use_public_resolver))
+ stack.enter_context(patch.object(dns.resolver, "default_resolver", None))
+ stack.enter_context(patch.object(dns.resolver, "time", self._clock))
+ yield self
+
+ def add_zone(self, apex: str, *, nameservers: dict[str, str] | None = None) -> None:
+ """Adds a zone served by `nameservers` (hostname to address). Without nameservers the zone only
+ answers through the public resolver."""
+ nameservers = nameservers or {}
+ self._zones.append(_Zone(apex=_absolute(apex), nameserver_addresses=tuple(nameservers.values())))
+ self.publish(apex, "SOA", f"ns.{apex}. hostmaster.{apex}. 1 7200 3600 1209600 300")
+ for hostname, address in nameservers.items():
+ self.publish(apex, "NS", f"{hostname}.")
+ if self._zone_containing(_absolute(hostname)) is None:
+ self.add_zone(hostname.split(".", 1)[1])
+ self.publish(hostname, "A", address)
+
+ def publish(self, name: str, rdtype: str, *values: str) -> None:
+ records = self._records.setdefault((_absolute(name), dns.rdatatype.from_text(rdtype)), [])
+ records.extend(
+ dns.rdata.from_text(dns.rdataclass.IN, rdtype, value, origin=dns.name.root, relativize=False)
+ for value in values
+ )
+
+ def publish_txt(self, name: str, *values: str) -> None:
+ self.publish(name, "TXT", *(f'"{value}"' for value in values))
+
+ def unpublish(self, name: str, rdtype: str) -> None:
+ self._records.pop((_absolute(name), dns.rdatatype.from_text(rdtype)), None)
+
+ def answers(self, name: str, rdtype: str) -> list[dns.rdata.Rdata]:
+ """What a public resolver returns for the lookup, following CNAMEs. Empty when nothing answers."""
+ rdtype_value = dns.rdatatype.from_text(rdtype)
+ resolution = self._resolve(_absolute(name), rdtype_value, within=None)
+ return [rdata for rrset in resolution.answer if rrset.rdtype == rdtype_value for rdata in rrset]
+
+ def txt_values(self, name: str) -> list[str]:
+ return [_txt_value(rdata) for rdata in self.answers(name, "TXT") if isinstance(rdata, TXT)]
+
+ def cname_target(self, name: str) -> str | None:
+ targets = [rdata.target for rdata in self.answers(name, "CNAME") if isinstance(rdata, CNAME)]
+ return _relative_text(targets[0]) if targets else None
+
+ def _answer_query(
+ self, query: dns.message.Message, where: str, timeout: float | None = None, *args: Any, **kwargs: Any
+ ) -> dns.message.Message:
+ self.queried_addresses.add(where)
+ question = query.question[0]
+ lookup = self._lookup_at(where)
+ if lookup is None:
+ self._time_out(timeout)
+ name, rdtype = _relative_text(question.name), dns.rdatatype.to_text(question.rdtype)
+ fault = self._faults.fault_for(
+ dns_operation(name, rdtype, at="public" if lookup == "public" else "zone"), dns_operation(name, rdtype)
+ )
+ if fault == DnsFault.TIMEOUT:
+ self._time_out(timeout)
+
+ response = dns.message.make_response(query)
+ if fault in (DnsFault.SERVFAIL, DnsFault.NXDOMAIN):
+ response.set_rcode(dns.rcode.SERVFAIL if fault == DnsFault.SERVFAIL else dns.rcode.NXDOMAIN)
+ else:
+ resolution = self._resolve(question.name, question.rdtype, within=None if lookup == "public" else lookup)
+ response.set_rcode(resolution.rcode)
+ response.answer.extend(resolution.answer)
+ response.authority.extend(resolution.authority)
+ return dns.message.from_wire(response.to_wire())
+
+ def _lookup_at(self, address: str) -> _Zone | Literal["public"] | None:
+ if address == PUBLIC_RESOLVER_ADDRESS:
+ return "public"
+ return next((zone for zone in self._zones if address in zone.nameserver_addresses), None)
+
+ def _time_out(self, timeout: float | None) -> None:
+ self._clock.sleep(timeout or UNANSWERED_QUERY_SECONDS)
+ raise dns.exception.Timeout(timeout=timeout)
+
+ def _resolve(self, name: dns.name.Name, rdtype: dns.rdatatype.RdataType, *, within: _Zone | None) -> _Resolution:
+ chain: list[dns.rrset.RRset] = []
+ for _ in range(MAX_CNAME_CHAIN):
+ zone = self._zone_containing(name)
+ if zone is None or (within is not None and zone != within):
+ if chain:
+ return _Resolution(rcode=dns.rcode.NOERROR, answer=tuple(chain))
+ return _Resolution(rcode=dns.rcode.REFUSED if within else dns.rcode.NXDOMAIN)
+ if records := self._records.get((name, rdtype)):
+ return _Resolution(rcode=dns.rcode.NOERROR, answer=(*chain, _rrset(name, records)))
+ alias = self._records.get((name, dns.rdatatype.CNAME))
+ if not alias or rdtype == dns.rdatatype.CNAME or not isinstance(alias[0], CNAME):
+ rcode = dns.rcode.NOERROR if self._name_exists(name) else dns.rcode.NXDOMAIN
+ return _Resolution(rcode=rcode, answer=tuple(chain), authority=(self._soa(zone),))
+ chain.append(_rrset(name, alias))
+ name = alias[0].target
+ return _Resolution(rcode=dns.rcode.SERVFAIL)
+
+ def _zone_containing(self, name: dns.name.Name) -> _Zone | None:
+ candidates = [zone for zone in self._zones if name.is_subdomain(zone.apex)]
+ return max(candidates, key=lambda zone: len(zone.apex), default=None)
+
+ def _name_exists(self, name: dns.name.Name) -> bool:
+ return any(record_name.is_subdomain(name) for record_name, _ in self._records)
+
+ def _soa(self, zone: _Zone) -> dns.rrset.RRset:
+ return _rrset(zone.apex, self._records[(zone.apex, dns.rdatatype.SOA)])
+
+
+def _use_public_resolver(resolver: dns.resolver.Resolver, _resolv_conf: object) -> None:
+ resolver.nameservers = [PUBLIC_RESOLVER_ADDRESS]
+
+
+def _rrset(name: dns.name.Name, records: list[dns.rdata.Rdata]) -> dns.rrset.RRset:
+ return dns.rrset.from_rdata_list(name, TTL, records)
+
+
+def _absolute(name: str) -> dns.name.Name:
+ return dns.name.from_text(name.lower(), origin=dns.name.root)
+
+
+def _relative_text(name: dns.name.Name) -> str:
+ return name.to_text(omit_final_dot=True)
+
+
+def _txt_value(rdata: TXT) -> str:
+ return b"".join(rdata.strings).decode()
diff --git a/products/workflows/backend/tests/fakes/fake_domain_connect.py b/products/workflows/backend/tests/fakes/fake_domain_connect.py
new file mode 100644
index 000000000000..b968a1e4b11a
--- /dev/null
+++ b/products/workflows/backend/tests/fakes/fake_domain_connect.py
@@ -0,0 +1,178 @@
+import re
+import json
+import base64
+from collections.abc import Iterator
+from contextlib import contextmanager
+from pathlib import Path
+from typing import Any
+from urllib.parse import parse_qs, urlsplit
+
+from django.conf import settings
+
+import requests
+import responses
+from cryptography.exceptions import InvalidSignature
+from cryptography.hazmat.primitives import hashes, serialization
+from cryptography.hazmat.primitives.asymmetric import padding
+from cryptography.hazmat.primitives.asymmetric.rsa import RSAPrivateKey, RSAPublicKey
+
+from products.workflows.backend.tests.fakes.fake_dns import FakeDnsZone
+from products.workflows.backend.tests.fakes.faults import FaultInjector, HttpFault
+
+TEMPLATE_DIR = (
+ Path(__file__).resolve().parents[5] / "frontend" / "src" / "lib" / "components" / "DomainConnect" / "templates"
+)
+CLOUDFLARE_ENDPOINT = "api.cloudflare.com/client/v4/dns/domainconnect"
+CLOUDFLARE_SYNC_UX = "https://dash.cloudflare.com/domainconnect"
+SETTINGS_OPERATION = "http.get:settings"
+_TEMPLATE_VARIABLE = re.compile(r"%(\w+)%")
+_SETTINGS_PATH = re.compile(r"/v2/(?P<domain>[^/]+)/settings$")
+_APPLY_PATH = re.compile(r"/v2/domainTemplates/providers/(?P<provider>[^/]+)/services/(?P<service>[^/]+)/apply$")
+
+
+class DomainConnectRejected(Exception):
+ pass
+
+
+class FakeDomainConnectProvider:
+ """A Domain Connect DNS provider. Discovery reads the `_domainconnect` TXT record from `FakeDnsZone`, settings
+ are served over HTTP, and `apply` checks the signature, renders our real template and publishes its records."""
+
+ def __init__(
+ self,
+ dns: FakeDnsZone,
+ faults: FaultInjector,
+ *,
+ endpoint: str = CLOUDFLARE_ENDPOINT,
+ url_sync_ux: str = CLOUDFLARE_SYNC_UX,
+ ) -> None:
+ self._dns = dns
+ self._faults = faults
+ self.endpoint = endpoint
+ self.url_sync_ux = url_sync_ux
+ self.hosted_domains: set[str] = set()
+
+ @contextmanager
+ def installed(self) -> Iterator["FakeDomainConnectProvider"]:
+ with responses.RequestsMock(assert_all_requests_are_fired=False) as http:
+ http.add_callback(
+ responses.GET, re.compile(rf"https://{re.escape(self.endpoint)}/v2/[^/]+/settings"), self._settings
+ )
+ yield self
+
+ def host(self, root_domain: str) -> None:
+ self.hosted_domains.add(root_domain)
+ self._dns.publish_txt(f"_domainconnect.{root_domain}", self.endpoint)
+
+ def apply(self, url: str) -> str | None:
+ """Approves the apply URL the way a person does at the provider, and returns where it redirects."""
+ parts = urlsplit(url)
+ if not url.startswith(self.url_sync_ux):
+ raise DomainConnectRejected(f"{url} is not this provider's sync URL")
+ self._check_signature(parts.query)
+ path = _APPLY_PATH.search(parts.path)
+ if path is None:
+ raise DomainConnectRejected(f"Unknown apply path {parts.path}")
+ params = {key: values[0] for key, values in parse_qs(parts.query).items()}
+ if params["domain"] not in self.hosted_domains:
+ raise DomainConnectRejected(f"{params['domain']} is not hosted here")
+ template = _load_template(path["provider"], path["service"])
+ redirect_uri = params.get("redirect_uri")
+ if redirect_uri and urlsplit(redirect_uri).hostname != template["syncRedirectDomain"]:
+ raise DomainConnectRejected(f"{redirect_uri} is outside the template's redirect domain")
+ selected_groups = set(params["groupId"].split(",")) if "groupId" in params else None
+ for record in template["records"]:
+ if selected_groups is None or record.get("groupId") in selected_groups:
+ self._publish(_render(record, params), domain=params["domain"], host=params.get("host", ""))
+ return redirect_uri
+
+ def _settings(self, request: requests.PreparedRequest) -> tuple[int, dict[str, str], str]:
+ settings_path = _SETTINGS_PATH.search(urlsplit(request.url or "").path)
+ domain = settings_path["domain"] if settings_path else ""
+ match self._faults.fault_for(SETTINGS_OPERATION):
+ case HttpFault.TIMEOUT:
+ raise requests.exceptions.ConnectTimeout(f"Injected timeout for {request.url}")
+ case HttpFault.SERVER_ERROR:
+ return 500, {}, "Internal Server Error"
+ case HttpFault.RATE_LIMITED:
+ return 429, {"Retry-After": "30"}, "Too Many Requests"
+ case HttpFault.MALFORMED_JSON:
+ return 200, {"Content-Type": "application/json"}, "{not json"
+ case HttpFault.MISSING_URL_SYNC_UX:
+ return 200, {"Content-Type": "application/json"}, json.dumps({"providerName": "Cloudflare"})
+ if domain not in self.hosted_domains:
+ return 404, {}, ""
+ body = {"providerId": "cloudflare.com", "providerName": "Cloudflare", "urlSyncUX": self.url_sync_ux}
+ return 200, {"Content-Type": "application/json"}, json.dumps(body)
+
+ def _check_signature(self, query: str) -> None:
+ signed_query, separator, signature_query = query.partition("&sig=")
+ if not separator:
+ raise DomainConnectRejected("The apply URL is not signed")
+ signature = parse_qs(f"sig={signature_query}")
+ if signature["key"][0] != settings.DOMAIN_CONNECT_KEY_ID:
+ raise DomainConnectRejected(f"Unknown signing key {signature['key'][0]}")
+ try:
+ _public_key().verify(
+ base64.b64decode(signature["sig"][0]), signed_query.encode(), padding.PKCS1v15(), hashes.SHA256()
+ )
+ except InvalidSignature as error:
+ raise DomainConnectRejected("The apply URL signature does not match") from error
+
+ def _publish(self, record: dict[str, Any], *, domain: str, host: str) -> None:
+ name = _record_name(record["host"], domain=domain, host=host)
+ match record["type"]:
+ case "TXT":
+ self._dns.publish_txt(name, record["data"])
+ case "CNAME":
+ self._dns.unpublish(name, "CNAME")
+ self._dns.publish(name, "CNAME", f"{record['pointsTo']}.")
+ case "MX":
+ self._dns.publish(name, "MX", f"{record['priority']} {record['pointsTo']}.")
+ case "SPFM":
+ self._merge_spf_rules(name, record["spfRules"])
+ case unknown:
+ raise DomainConnectRejected(f"Unsupported record type {unknown}")
+
+ def _merge_spf_rules(self, name: str, rules: str) -> None:
+ existing = next((value for value in self._dns.txt_values(name) if value.startswith("v=spf1")), None)
+ if existing is None:
+ self._dns.publish_txt(name, f"v=spf1 {rules} ~all")
+ return
+ terms = existing.split()
+ merged = [terms[0], *(rule for rule in rules.split() if rule not in terms), *terms[1:]]
+ others = [value for value in self._dns.txt_values(name) if value != existing]
+ self._dns.unpublish(name, "TXT")
+ self._dns.publish_txt(name, *others, " ".join(merged))
+
+
+def _record_name(record_host: str, *, domain: str, host: str) -> str:
+ labels = [label for label in (record_host if record_host != "@" else "", host) if label]
+ return ".".join([*labels, domain])
+
+
+def _render(record: dict[str, Any], variables: dict[str, str]) -> dict[str, Any]:
+ def substitute(value: Any) -> Any:
+ if not isinstance(value, str):
+ return value
+ missing = set(_TEMPLATE_VARIABLE.findall(value)) - variables.keys()
+ if missing:
+ raise DomainConnectRejected(f"The apply URL is missing template variables {sorted(missing)}")
+ return _TEMPLATE_VARIABLE.sub(lambda match: variables[match[1]], value)
+
+ return {key: substitute(value) for key, value in record.items()}
+
+
+def _load_template(provider_id: str, service_id: str) -> dict[str, Any]:
+ path = TEMPLATE_DIR / f"{provider_id}.{service_id}.json"
+ if not path.exists():
+ raise DomainConnectRejected(f"No template {provider_id}/{service_id}")
+ return json.loads(path.read_text())
+
+
+def _public_key() -> RSAPublicKey:
+ if not settings.DOMAIN_CONNECT_PRIVATE_KEY:
+ raise DomainConnectRejected("PostHog has no published signing key")
+ private_key = serialization.load_pem_private_key(settings.DOMAIN_CONNECT_PRIVATE_KEY.encode(), password=None)
+ assert isinstance(private_key, RSAPrivateKey)
+ return private_key.public_key()
diff --git a/products/workflows/backend/tests/fakes/fake_ses.py b/products/workflows/backend/tests/fakes/fake_ses.py
new file mode 100644
index 000000000000..d4123accf477
--- /dev/null
+++ b/products/workflows/backend/tests/fakes/fake_ses.py
@@ -0,0 +1,232 @@
+from collections.abc import Callable, Iterator
+from contextlib import contextmanager
+from typing import Any
+
+from unittest.mock import patch
+
+from botocore.exceptions import ClientError
+from dns.rdtypes.ANY.MX import MX
+
+from posthog.dataclasses import frozen
+
+from products.workflows.backend.tests.fakes.fake_dns import FakeDnsZone
+from products.workflows.backend.tests.fakes.faults import AwsFault, FaultInjector
+
+ACCOUNT_ID = "123456789012"
+SUCCESS = "Success"
+PENDING = "Pending"
+
+
+@frozen(frozen=False)
+class SesIdentity:
+ domain: str
+ verification_token: str | None
+ dkim_tokens: tuple[str, ...] = ()
+ mail_from_domain: str | None = None
+ forced_verification_status: str | None = None
+ forced_dkim_status: str | None = None
+ checks_paused: bool = False
+ feedback_forwarding: bool = True
+
+
+class FakeSes:
+ """SES v1, SESv2 tenants and STS behind `boto3.client`. Each read checks `FakeDnsZone` the way SES polls
+ DNS, so a part flips to Success only once its records are published."""
+
+ def __init__(self, dns: FakeDnsZone, faults: FaultInjector, *, region: str = "us-east-1") -> None:
+ self._dns = dns
+ self._faults = faults
+ self.region = region
+ self.identities: dict[str, SesIdentity] = {}
+ self.tenant_resources: dict[str, set[str]] = {}
+ self.writes: list[str] = []
+
+ @contextmanager
+ def installed(self) -> Iterator["FakeSes"]:
+ with patch("boto3.client", self._client):
+ yield self
+
+ def add_identity(self, domain: str, **fields: Any) -> SesIdentity:
+ identity = SesIdentity(domain=domain, **{"verification_token": f"token-{domain}", **fields})
+ self.identities[domain] = identity
+ return identity
+
+ def associate(self, domain: str, tenant: str) -> None:
+ self.tenant_resources.setdefault(tenant, set()).add(self.identity_arn(domain))
+
+ def transfer_identity(self, domain: str, tenant: str) -> None:
+ for resources in self.tenant_resources.values():
+ resources.discard(self.identity_arn(domain))
+ self.associate(domain, tenant)
+
+ def tenants_of(self, domain: str) -> set[str]:
+ return {tenant for tenant, resources in self.tenant_resources.items() if self.identity_arn(domain) in resources}
+
+ def identity_arn(self, domain: str) -> str:
+ return f"arn:aws:ses:{self.region}:{ACCOUNT_ID}:identity/{domain}"
+
+ def verification_status(self, identity: SesIdentity) -> str:
+ if identity.checks_paused:
+ return PENDING
+ if identity.forced_verification_status:
+ return identity.forced_verification_status
+ published = identity.verification_token in self._dns.txt_values(f"_amazonses.{identity.domain}")
+ return SUCCESS if published else PENDING
+
+ def dkim_status(self, identity: SesIdentity) -> str:
+ if identity.checks_paused:
+ return PENDING
+ if identity.forced_dkim_status:
+ return identity.forced_dkim_status
+ if not identity.dkim_tokens:
+ return "NotStarted"
+ published = all(
+ self._dns.cname_target(f"{token}._domainkey.{identity.domain}") == f"{token}.dkim.amazonses.com"
+ for token in identity.dkim_tokens
+ )
+ return SUCCESS if published else PENDING
+
+ def mail_from_status(self, identity: SesIdentity) -> str:
+ assert identity.mail_from_domain is not None
+ if identity.checks_paused:
+ return PENDING
+ expected_exchange = f"feedback-smtp.{self.region}.amazonses.com."
+ exchanges = [
+ rdata.exchange.to_text()
+ for rdata in self._dns.answers(identity.mail_from_domain, "MX")
+ if isinstance(rdata, MX)
+ ]
+ # SES rejects a MAIL FROM domain that has any MX record besides its own.
+ return SUCCESS if exchanges == [expected_exchange] else PENDING
+
+ def _client(self, service_name: str, **_config: Any) -> "_FakeAwsClient":
+ handlers: dict[str, dict[str, Callable[..., Any]]] = {
+ "ses": {
+ "verify_domain_identity": self._verify_domain_identity,
+ "verify_domain_dkim": self._verify_domain_dkim,
+ "set_identity_mail_from_domain": self._set_identity_mail_from_domain,
+ "get_identity_verification_attributes": self._get_identity_verification_attributes,
+ "get_identity_dkim_attributes": self._get_identity_dkim_attributes,
+ "get_identity_mail_from_domain_attributes": self._get_identity_mail_from_domain_attributes,
+ },
+ "sesv2": {
+ "create_tenant": self._create_tenant,
+ "create_tenant_resource_association": self._create_tenant_resource_association,
+ "list_resource_tenants": self._list_resource_tenants,
+ "put_email_identity_feedback_attributes": self._put_email_identity_feedback_attributes,
+ },
+ "sts": {"get_caller_identity": lambda: {"Account": ACCOUNT_ID}},
+ }
+ if service_name not in handlers:
+ raise ValueError(f"FakeSes does not serve the {service_name} client")
+ return _FakeAwsClient(service_name, handlers[service_name], self._faults)
+
+ def _verify_domain_identity(self, *, Domain: str) -> dict[str, Any]:
+ self.writes.append("verify_domain_identity")
+ identity = self.identities.get(Domain) or self.add_identity(Domain)
+ return {"VerificationToken": identity.verification_token}
+
+ def _verify_domain_dkim(self, *, Domain: str) -> dict[str, Any]:
+ self.writes.append("verify_domain_dkim")
+ identity = self.identities.get(Domain) or self.add_identity(Domain)
+ if not identity.dkim_tokens:
+ identity.dkim_tokens = tuple(f"dk{index}{Domain.replace('.', '')}" for index in range(1, 4))
+ return {"DkimTokens": list(identity.dkim_tokens)}
+
+ def _set_identity_mail_from_domain(self, *, Identity: str, MailFromDomain: str, **_options: Any) -> dict:
+ self.writes.append("set_identity_mail_from_domain")
+ self._existing(Identity, "SetIdentityMailFromDomain").mail_from_domain = MailFromDomain
+ return {}
+
+ def _get_identity_verification_attributes(self, *, Identities: list[str]) -> dict[str, Any]:
+ return {
+ "VerificationAttributes": {
+ identity.domain: {
+ "VerificationStatus": self.verification_status(identity),
+ **({"VerificationToken": identity.verification_token} if identity.verification_token else {}),
+ }
+ for identity in self._known(Identities)
+ }
+ }
+
+ def _get_identity_dkim_attributes(self, *, Identities: list[str]) -> dict[str, Any]:
+ return {
+ "DkimAttributes": {
+ identity.domain: {
+ "DkimEnabled": bool(identity.dkim_tokens),
+ "DkimVerificationStatus": self.dkim_status(identity),
+ **({"DkimTokens": list(identity.dkim_tokens)} if identity.dkim_tokens else {}),
+ }
+ for identity in self._known(Identities)
+ }
+ }
+
+ def _get_identity_mail_from_domain_attributes(self, *, Identities: list[str]) -> dict[str, Any]:
+ return {
+ "MailFromDomainAttributes": {
+ identity.domain: {
+ "MailFromDomain": identity.mail_from_domain,
+ "MailFromDomainStatus": self.mail_from_status(identity),
+ "BehaviorOnMXFailure": "UseDefaultValue",
+ }
+ for identity in self._known(Identities)
+ if identity.mail_from_domain
+ }
+ }
+
+ def _put_email_identity_feedback_attributes(self, *, EmailIdentity: str, EmailForwardingEnabled: bool) -> dict:
+ self.writes.append("put_email_identity_feedback_attributes")
+ self._existing(EmailIdentity, "PutEmailIdentityFeedbackAttributes").feedback_forwarding = EmailForwardingEnabled
+ return {}
+
+ def _create_tenant(self, *, TenantName: str, **_options: Any) -> dict:
+ self.writes.append("create_tenant")
+ if TenantName in self.tenant_resources:
+ raise ClientError({"Error": {"Code": "AlreadyExistsException"}}, "CreateTenant")
+ self.tenant_resources[TenantName] = set()
+ return {"TenantName": TenantName}
+
+ def _create_tenant_resource_association(self, *, TenantName: str, ResourceArn: str) -> dict:
+ self.writes.append("create_tenant_resource_association")
+ resources = self.tenant_resources.setdefault(TenantName, set())
+ if ResourceArn in resources:
+ raise ClientError({"Error": {"Code": "AlreadyExistsException"}}, "CreateTenantResourceAssociation")
+ resources.add(ResourceArn)
+ return {}
+
+ def _list_resource_tenants(self, *, ResourceArn: str) -> dict[str, Any]:
+ return {
+ "ResourceTenants": [
+ {"TenantName": tenant, "ResourceArn": ResourceArn}
+ for tenant, resources in sorted(self.tenant_resources.items())
+ if ResourceArn in resources
+ ]
+ }
+
+ def _known(self, domains: list[str]) -> list[SesIdentity]:
+ return [self.identities[domain] for domain in domains if domain in self.identities]
+
+ def _existing(self, domain: str, operation: str) -> SesIdentity:
+ if domain not in self.identities:
+ raise ClientError({"Error": {"Code": "InvalidParameterValue"}}, operation)
+ return self.identities[domain]
+
+
+class _FakeAwsClient:
+ def __init__(self, service: str, handlers: dict[str, Callable[..., Any]], faults: FaultInjector) -> None:
+ self._service = service
+ self._handlers = handlers
+ self._faults = faults
+
+ def __getattr__(self, operation: str) -> Callable[..., Any]:
+ if operation not in self._handlers:
+ raise AttributeError(f"FakeSes {self._service} client has no {operation}")
+ handler = self._handlers[operation]
+
+ def call(**kwargs: Any) -> Any:
+ fault = self._faults.fault_for(f"{self._service}.{operation}")
+ if isinstance(fault, AwsFault):
+ raise fault.error(operation)
+ return handler(**kwargs)
+
+ return call
diff --git a/products/workflows/backend/tests/fakes/faults.py b/products/workflows/backend/tests/fakes/faults.py
new file mode 100644
index 000000000000..613db7503f50
--- /dev/null
+++ b/products/workflows/backend/tests/fakes/faults.py
@@ -0,0 +1,64 @@
+from enum import StrEnum
+
+from botocore.exceptions import ClientError, EndpointConnectionError, ReadTimeoutError
+
+AWS_ENDPOINT = "https://email.us-east-1.amazonaws.com"
+
+
+class AwsFault(StrEnum):
+ THROTTLING = "throttling"
+ ACCESS_DENIED = "access_denied"
+ INVALID_PARAMETER = "invalid_parameter"
+ ENDPOINT_UNREACHABLE = "endpoint_unreachable"
+ READ_TIMEOUT = "read_timeout"
+
+ def error(self, operation: str) -> Exception:
+ match self:
+ case AwsFault.THROTTLING:
+ return _client_error("Throttling", operation)
+ case AwsFault.ACCESS_DENIED:
+ return _client_error("AccessDenied", operation)
+ case AwsFault.INVALID_PARAMETER:
+ return _client_error("InvalidParameterValue", operation)
+ case AwsFault.ENDPOINT_UNREACHABLE:
+ return EndpointConnectionError(endpoint_url=AWS_ENDPOINT)
+ case AwsFault.READ_TIMEOUT:
+ return ReadTimeoutError(endpoint_url=AWS_ENDPOINT)
+
+
+class DnsFault(StrEnum):
+ TIMEOUT = "timeout"
+ SERVFAIL = "servfail"
+ NXDOMAIN = "nxdomain"
+
+
+class HttpFault(StrEnum):
+ SERVER_ERROR = "server_error"
+ RATE_LIMITED = "rate_limited"
+ TIMEOUT = "timeout"
+ MALFORMED_JSON = "malformed_json"
+ MISSING_URL_SYNC_UX = "missing_url_sync_ux"
+
+
+Fault = AwsFault | DnsFault | HttpFault
+
+
+class FaultInjector:
+ """Faults keyed by operation, such as `ses.get_identity_dkim_attributes`, `dns:zone:_dmarc.example.com/TXT`
+ or `http.get:settings`. A fault holds until it is cleared, so retries inside a client see it too."""
+
+ def __init__(self) -> None:
+ self._faults: dict[str, Fault] = {}
+
+ def inject(self, operation: str, fault: Fault) -> None:
+ self._faults[operation] = fault
+
+ def clear(self) -> None:
+ self._faults.clear()
+
+ def fault_for(self, *operations: str) -> Fault | None:
+ return next((self._faults[operation] for operation in operations if operation in self._faults), None)
+
+
+def _client_error(code: str, operation: str) -> ClientError:
+ return ClientError({"Error": {"Code": code, "Message": f"Injected {code}"}}, operation)
diff --git a/products/workflows/backend/tests/fakes/world.py b/products/workflows/backend/tests/fakes/world.py
new file mode 100644
index 000000000000..24cf52245183
--- /dev/null
+++ b/products/workflows/backend/tests/fakes/world.py
@@ -0,0 +1,46 @@
+from collections.abc import Iterator, Mapping
+from contextlib import contextmanager
+from functools import cache
+from typing import Any
+
+from cryptography.hazmat.primitives import serialization
+from cryptography.hazmat.primitives.asymmetric import rsa
+
+from products.workflows.backend.tests.fakes.fake_dns import FakeDnsZone
+from products.workflows.backend.tests.fakes.fake_domain_connect import FakeDomainConnectProvider
+from products.workflows.backend.tests.fakes.fake_ses import FakeSes
+from products.workflows.backend.tests.fakes.faults import FaultInjector
+
+
+class EmailDomainWorld:
+ """Everything outside PostHog that the email domain setup talks to, sharing one DNS and one fault injector."""
+
+ def __init__(self) -> None:
+ self.faults = FaultInjector()
+ self.dns = FakeDnsZone(self.faults)
+ self.ses = FakeSes(self.dns, self.faults)
+ self.domain_connect = FakeDomainConnectProvider(self.dns, self.faults)
+
+ @contextmanager
+ def installed(self) -> Iterator["EmailDomainWorld"]:
+ with self.dns.installed(), self.ses.installed(), self.domain_connect.installed():
+ yield self
+
+ def publish_record(self, record: Mapping[str, Any]) -> None:
+ """Publishes one record from the status API the way a person copies it into their DNS host."""
+ hostname, value = record["hostname"], record["value"]
+ match record["type"]:
+ case "TXT":
+ self.dns.publish_txt(hostname, value)
+ case "CNAME":
+ self.dns.publish(hostname, "CNAME", f"{value}.")
+ case "MX":
+ self.dns.publish(hostname, "MX", f"{record['priority']} {value}.")
+
+
+@cache
+def domain_connect_signing_key_pem() -> str:
+ key = rsa.generate_private_key(public_exponent=65537, key_size=2048)
+ return key.private_bytes(
+ serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8, serialization.NoEncryption()
+ ).decode()
diff --git a/services/mcp/schema/generated-tool-definitions.json b/services/mcp/schema/generated-tool-definitions.json
index 6eea11b3c6ab..fe6ff27c5943 100644
--- a/services/mcp/schema/generated-tool-definitions.json
+++ b/services/mcp/schema/generated-tool-definitions.json
@@ -8246,6 +8246,38 @@
"readOnlyHint": true
}
},
+ "integrations-email-status-retrieve": {
+ "description": "Get the setup progress of an email sender's domain by integration ID. Returns the overall status, the three setup steps, and every DNS record to publish with its current status (pending, found, verified). Poll it after adding DNS records until status is verified, waiting at least 10 seconds between calls. Pass refresh=true right after changing DNS to skip the 10 second cache. It does not change DNS or SES, and it does not mark the sender verified: once status is verified, call integrations-email-verify-create.",
+ "category": "Integrations",
+ "feature": "integrations",
+ "summary": "Get email sending domain status",
+ "title": "Get email sending domain status",
+ "required_scopes": ["integration:read"],
+ "api_operations": ["GET /api/projects/{project_id}/integrations/{id}/email/status/"],
+ "annotations": {
+ "destructiveHint": false,
+ "idempotentHint": true,
+ "openWorldHint": true,
+ "readOnlyHint": true
+ },
+ "feature_flag": "workflows-email-domain-agent-setup"
+ },
+ "integrations-email-verify-create": {
+ "description": "Record that an email sender's domain is verified, by integration ID, so its senders can send. Call it once integrations-email-status-retrieve reports status verified. It asks SES to check the domain again and returns status success when SES has verified it. Needs a project admin.",
+ "category": "Integrations",
+ "feature": "integrations",
+ "summary": "Verify email sending domain",
+ "title": "Verify email sending domain",
+ "required_scopes": ["integration:write"],
+ "api_operations": ["POST /api/projects/{project_id}/integrations/{id}/email/verify/"],
+ "annotations": {
+ "destructiveHint": false,
+ "idempotentHint": true,
+ "openWorldHint": true,
+ "readOnlyHint": false
+ },
+ "feature_flag": "workflows-email-domain-agent-setup"
+ },
"integrations-github-repos-retrieve": {
"description": "List the repositories accessible to a GitHub integration (by integration ID). Serves the cached repository list from the GitHub App installation. Supports `search` (case-insensitive name match), `limit` (default 100, max 500), and `offset` query parameters. Returns repository id, name, and full_name (owner/repo), a has_more flag, a total count, and next_offset. When has_more is true, call again with `offset` set to next_offset to get the next page. Set `compact` to false only when you need the extra repository details. Use this to validate that a repository is accessible to the integration before referencing it (e.g. when creating a GitHub data warehouse source).",
"category": "Integrations",
diff --git a/services/mcp/schema/tool-definitions-all.json b/services/mcp/schema/tool-definitions-all.json
index 14bf3a03b498..dc508bfda9dd 100644
--- a/services/mcp/schema/tool-definitions-all.json
+++ b/services/mcp/schema/tool-definitions-all.json
@@ -8483,6 +8483,38 @@
"readOnlyHint": true
}
},
+ "integrations-email-status-retrieve": {
+ "description": "Get the setup progress of an email sender's domain by integration ID. Returns the overall status, the three setup steps, and every DNS record to publish with its current status (pending, found, verified). Poll it after adding DNS records until status is verified, waiting at least 10 seconds between calls. Pass refresh=true right after changing DNS to skip the 10 second cache. It does not change DNS or SES, and it does not mark the sender verified: once status is verified, call integrations-email-verify-create.",
+ "category": "Integrations",
+ "feature": "integrations",
+ "summary": "Get email sending domain status",
+ "title": "Get email sending domain status",
+ "required_scopes": ["integration:read"],
+ "api_operations": ["GET /api/projects/{project_id}/integrations/{id}/email/status/"],
+ "annotations": {
+ "destructiveHint": false,
+ "idempotentHint": true,
+ "openWorldHint": true,
+ "readOnlyHint": true
+ },
+ "feature_flag": "workflows-email-domain-agent-setup"
+ },
+ "integrations-email-verify-create": {
+ "description": "Record that an email sender's domain is verified, by integration ID, so its senders can send. Call it once integrations-email-status-retrieve reports status verified. It asks SES to check the domain again and returns status success when SES has verified it. Needs a project admin.",
+ "category": "Integrations",
+ "feature": "integrations",
+ "summary": "Verify email sending domain",
+ "title": "Verify email sending domain",
+ "required_scopes": ["integration:write"],
+ "api_operations": ["POST /api/projects/{project_id}/integrations/{id}/email/verify/"],
+ "annotations": {
+ "destructiveHint": false,
+ "idempotentHint": true,
+ "openWorldHint": true,
+ "readOnlyHint": false
+ },
+ "feature_flag": "workflows-email-domain-agent-setup"
+ },
"integrations-github-repos-retrieve": {
"description": "List the repositories accessible to a GitHub integration (by integration ID). Serves the cached repository list from the GitHub App installation. Supports `search` (case-insensitive name match), `limit` (default 100, max 500), and `offset` query parameters. Returns repository id, name, and full_name (owner/repo), a has_more flag, a total count, and next_offset. When has_more is true, call again with `offset` set to next_offset to get the next page. Set `compact` to false only when you need the extra repository details. Use this to validate that a repository is accessible to the integration before referencing it (e.g. when creating a GitHub data warehouse source).",
"category": "Integrations",
diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts
index e50999289499..9c0a81d36b02 100644
--- a/services/mcp/src/api/generated.ts
+++ b/services/mcp/src/api/generated.ts
@@ -40996,6 +40996,210 @@ export namespace Schemas {
name: string;
}
+ /**
+ * * `verification` - Domain ownership
+ * * `dkim` - DKIM signing
+ * * `spf` - SPF
+ * * `mail_from_mx` - MAIL FROM MX
+ * * `mail_from_spf` - MAIL FROM SPF
+ * * `dmarc` - DMARC
+ */
+ export type EmailDomainSetupRecordKindEnum = typeof EmailDomainSetupRecordKindEnum[keyof typeof EmailDomainSetupRecordKindEnum];
+
+
+ export const EmailDomainSetupRecordKindEnum = {
+ Verification: 'verification',
+ Dkim: 'dkim',
+ Spf: 'spf',
+ MailFromMx: 'mail_from_mx',
+ MailFromSpf: 'mail_from_spf',
+ Dmarc: 'dmarc',
+ } as const;
+
+ /**
+ * * `pending` - Not found yet
+ * * `found` - Found in DNS
+ * * `verified` - Verified
+ */
+ export type EmailDomainSetupRecordStatusEnum = typeof EmailDomainSetupRecordStatusEnum[keyof typeof EmailDomainSetupRecordStatusEnum];
+
+
+ export const EmailDomainSetupRecordStatusEnum = {
+ Pending: 'pending',
+ Found: 'found',
+ Verified: 'verified',
+ } as const;
+
+ /**
+ * * `TXT` - TXT
+ * * `CNAME` - CNAME
+ * * `MX` - MX
+ */
+ export type EmailDomainSetupRecordTypeEnum = typeof EmailDomainSetupRecordTypeEnum[keyof typeof EmailDomainSetupRecordTypeEnum];
+
+
+ export const EmailDomainSetupRecordTypeEnum = {
+ Txt: 'TXT',
+ Cname: 'CNAME',
+ Mx: 'MX',
+ } as const;
+
+ /**
+ * * `not_started` - Not started
+ * * `pending` - Waiting for DNS records
+ * * `records_found` - Records found, waiting for verification
+ * * `verified` - Verified
+ * * `temporary_failure` - Temporary failure
+ * * `failed` - Failed
+ */
+ export type EmailDomainSetupStatusEnum = typeof EmailDomainSetupStatusEnum[keyof typeof EmailDomainSetupStatusEnum];
+
+
+ export const EmailDomainSetupStatusEnum = {
+ NotStarted: 'not_started',
+ Pending: 'pending',
+ RecordsFound: 'records_found',
+ Verified: 'verified',
+ TemporaryFailure: 'temporary_failure',
+ Failed: 'failed',
+ } as const;
+
+ /**
+ * * `domain_added` - Domain added
+ * * `records_found` - DNS records found
+ * * `verified` - Verified
+ */
+ export type EmailDomainSetupStepKeyEnum = typeof EmailDomainSetupStepKeyEnum[keyof typeof EmailDomainSetupStepKeyEnum];
+
+
+ export const EmailDomainSetupStepKeyEnum = {
+ DomainAdded: 'domain_added',
+ RecordsFound: 'records_found',
+ Verified: 'verified',
+ } as const;
+
+ /**
+ * * `done` - Done
+ * * `pending` - Pending
+ * * `failed` - Failed
+ */
+ export type EmailDomainSetupStepStateEnum = typeof EmailDomainSetupStepStateEnum[keyof typeof EmailDomainSetupStepStateEnum];
+
+
+ export const EmailDomainSetupStepStateEnum = {
+ Done: 'done',
+ Pending: 'pending',
+ Failed: 'failed',
+ } as const;
+
+ export interface EmailDomainStatusStep {
+ /** The setup step: domain_added (SES knows the domain), records_found (every DNS record is published), verified (SES verified the domain and the sender can send).
+ *
+ * * `domain_added` - Domain added
+ * * `records_found` - DNS records found
+ * * `verified` - Verified */
+ key: EmailDomainSetupStepKeyEnum;
+ /** Progress of this step.
+ *
+ * * `done` - Done
+ * * `pending` - Pending
+ * * `failed` - Failed */
+ state: EmailDomainSetupStepStateEnum;
+ }
+
+ export interface EmailDomainStatusRecord {
+ /** What the record does: verification proves domain ownership, dkim signs email, spf and mail_from_spf list SES as an allowed sender, mail_from_mx routes bounces, dmarc sets the policy for unauthenticated mail.
+ *
+ * * `verification` - Domain ownership
+ * * `dkim` - DKIM signing
+ * * `spf` - SPF
+ * * `mail_from_mx` - MAIL FROM MX
+ * * `mail_from_spf` - MAIL FROM SPF
+ * * `dmarc` - DMARC */
+ kind: EmailDomainSetupRecordKindEnum;
+ /** Full DNS name to create the record at, including the domain. */
+ hostname: string;
+ /** DNS record type.
+ *
+ * * `TXT` - TXT
+ * * `CNAME` - CNAME
+ * * `MX` - MX */
+ type: EmailDomainSetupRecordTypeEnum;
+ /** Value to publish. For an SPF record, add the include to an existing SPF record instead of creating a second one. For DMARC, any existing v=DMARC1 record counts. */
+ value: string;
+ /**
+ * MX priority. Null for other record types.
+ * @nullable
+ */
+ priority: number | null;
+ /** pending: not found in DNS yet. found: published in DNS, SES has not confirmed it yet. verified: SES confirmed the record, or for DMARC a v=DMARC1 record exists.
+ *
+ * * `pending` - Not found yet
+ * * `found` - Found in DNS
+ * * `verified` - Verified */
+ status: EmailDomainSetupRecordStatusEnum;
+ }
+
+ export interface EmailDomainStatus {
+ /** Overall setup status. records_found means every record is published and SES has not confirmed yet. temporary_failure means SES could not see the records, usually because DNS changes have not spread yet. failed means SES gave up after 72 hours; start verification again.
+ *
+ * * `not_started` - Not started
+ * * `pending` - Waiting for DNS records
+ * * `records_found` - Records found, waiting for verification
+ * * `verified` - Verified
+ * * `temporary_failure` - Temporary failure
+ * * `failed` - Failed */
+ status: EmailDomainSetupStatusEnum;
+ /** Whether the sender can send email. Set by email/verify once SES verifies the domain. */
+ verified: boolean;
+ /** When SES and DNS were read. Responses are cached for 10 seconds unless refresh is set. */
+ checked_at: string;
+ /** The three setup steps, in order. */
+ steps: EmailDomainStatusStep[];
+ /** DNS records to publish at the domain's DNS host, with the status of each. Empty when status is not_started. */
+ records: EmailDomainStatusRecord[];
+ }
+
+ /**
+ * * `success` - Success
+ * * `pending` - Pending
+ * * `failed` - Failed
+ */
+ export type EmailDomainVerificationStatusEnum = typeof EmailDomainVerificationStatusEnum[keyof typeof EmailDomainVerificationStatusEnum];
+
+
+ export const EmailDomainVerificationStatusEnum = {
+ Success: 'success',
+ Pending: 'pending',
+ Failed: 'failed',
+ } as const;
+
+ export interface EmailDomainVerificationRecord {
+ /** What the record does: verification, dkim, mail_from or dmarc. */
+ type: string;
+ /** DNS record type: TXT, CNAME or MX. */
+ recordType: string;
+ /** DNS name to create the record at. */
+ recordHostname: string;
+ /** Value to publish. */
+ recordValue: string;
+ /** success once SES or DNS confirms the record, else pending. */
+ status: string;
+ /** MX priority. Only set on MX records. */
+ priority?: number;
+ }
+
+ export interface EmailDomainVerification {
+ /** success when SES verified the domain and its senders are now marked verified.
+ *
+ * * `success` - Success
+ * * `pending` - Pending
+ * * `failed` - Failed */
+ status: EmailDomainVerificationStatusEnum;
+ /** DNS records the domain needs. */
+ dnsRecords: EmailDomainVerificationRecord[];
+ }
+
/**
* * `ai` - AI
* * `template` - Template
@@ -134669,6 +134873,13 @@ export namespace Schemas {
search?: string;
};
+ export type IntegrationsEmailStatusRetrieveParams = {
+ /**
+ * Skip the 10 second status cache and read SES and DNS again. Use it after changing DNS records.
+ */
+ refresh?: boolean;
+ };
+
export type IntegrationsGithubAssigneesRetrieveParams = {
/**
* Repository name, or owner/name, whose assignable users to list.
diff --git a/services/mcp/src/generated/integrations/api.ts b/services/mcp/src/generated/integrations/api.ts
index 89932c5bc6dc..54a64cccccd5 100644
--- a/services/mcp/src/generated/integrations/api.ts
+++ b/services/mcp/src/generated/integrations/api.ts
@@ -3,7 +3,7 @@
* MCP service uses these Zod schemas for generated tool handlers.
* To regenerate: hogli build:openapi
*
- * PostHog API - MCP 9 enabled ops
+ * PostHog API - MCP 11 enabled ops
* OpenAPI spec version: 1.0.0
*/
import * as zod from 'zod'
@@ -145,6 +145,41 @@ export const IntegrationsChannelsRetrieveQueryParams = () => zod.object({
.describe('Optional case-insensitive channel name or ID search query.'),
})
+/**
+ * Read the setup progress of an email sender's domain from SES and public DNS without changing either. Poll this until status is verified, then call email/verify once so the sender can send.
+ * @summary Get email sending domain status
+ */
+export const IntegrationsEmailStatusRetrieveParams = () => zod.object({
+ id: zod.number().describe('A unique integer value identifying this integration.'),
+ project_id: zod
+ .string()
+ .describe(
+ "Project ID of the project you're trying to access. To find the ID of the project, make a call to \/api\/projects\/."
+ ),
+})
+
+export const integrationsEmailStatusRetrieveQueryRefreshDefault = false
+
+export const IntegrationsEmailStatusRetrieveQueryParams = () => zod.object({
+ refresh: zod
+ .boolean()
+ .default(integrationsEmailStatusRetrieveQueryRefreshDefault)
+ .describe('Skip the 10 second status cache and read SES and DNS again. Use it after changing DNS records.'),
+})
+
+/**
+ * Ask SES to check the domain's DNS records again. When SES has verified the domain, mark the domain's senders as verified so they can send. Call it once email/status reports status verified.
+ * @summary Verify an email sending domain
+ */
+export const IntegrationsEmailVerifyCreateParams = () => zod.object({
+ id: zod.number().describe('A unique integer value identifying this integration.'),
+ project_id: zod
+ .string()
+ .describe(
+ "Project ID of the project you're trying to access. To find the ID of the project, make a call to \/api\/projects\/."
+ ),
+})
+
export const IntegrationsGithubReposRetrieveParams = () => zod.object({
id: zod.number().describe('A unique integer value identifying this integration.'),
project_id: zod
diff --git a/services/mcp/src/tools/generated/integrations.ts b/services/mcp/src/tools/generated/integrations.ts
index eaeb379c6f1d..e2a01775dede 100644
--- a/services/mcp/src/tools/generated/integrations.ts
+++ b/services/mcp/src/tools/generated/integrations.ts
@@ -81,6 +81,54 @@ const integrationsChannelsRetrieve = (): ToolBase<
},
})
+const IntegrationsEmailStatusRetrieveSchema = () => {
+ const IntegrationsEmailStatusRetrieveParams = orvalSchemas.IntegrationsEmailStatusRetrieveParams()
+ const IntegrationsEmailStatusRetrieveQueryParams = orvalSchemas.IntegrationsEmailStatusRetrieveQueryParams()
+ return IntegrationsEmailStatusRetrieveParams.omit({ project_id: true }).extend(
+ IntegrationsEmailStatusRetrieveQueryParams.shape
+ )
+}
+
+const integrationsEmailStatusRetrieve = (): ToolBase<
+ ReturnType<typeof IntegrationsEmailStatusRetrieveSchema>,
+ Schemas.EmailDomainStatus
+> => ({
+ name: 'integrations-email-status-retrieve',
+ schema: IntegrationsEmailStatusRetrieveSchema(),
+ handler: async (context: Context, params: z.infer<ReturnType<typeof IntegrationsEmailStatusRetrieveSchema>>) => {
+ const projectId = await context.stateManager.getProjectId()
+ const result = await context.api.request<Schemas.EmailDomainStatus>({
+ method: 'GET',
+ path: `/api/projects/${encodeURIComponent(String(projectId))}/integrations/${encodeURIComponent(String(params.id))}/email/status/`,
+ query: {
+ refresh: params.refresh,
+ },
+ })
+ return result
+ },
+})
+
+const IntegrationsEmailVerifyCreateSchema = () => {
+ const IntegrationsEmailVerifyCreateParams = orvalSchemas.IntegrationsEmailVerifyCreateParams()
+ return IntegrationsEmailVerifyCreateParams.omit({ project_id: true })
+}
+
+const integrationsEmailVerifyCreate = (): ToolBase<
+ ReturnType<typeof IntegrationsEmailVerifyCreateSchema>,
+ Schemas.EmailDomainVerification
+> => ({
+ name: 'integrations-email-verify-create',
+ schema: IntegrationsEmailVerifyCreateSchema(),
+ handler: async (context: Context, params: z.infer<ReturnType<typeof IntegrationsEmailVerifyCreateSchema>>) => {
+ const projectId = await context.stateManager.getProjectId()
+ const result = await context.api.request<Schemas.EmailDomainVerification>({
+ method: 'POST',
+ path: `/api/projects/${encodeURIComponent(String(projectId))}/integrations/${encodeURIComponent(String(params.id))}/email/verify/`,
+ })
+ return result
+ },
+})
+
const IntegrationsGithubReposRetrieveSchema = () => {
const IntegrationsGithubReposRetrieveParams = orvalSchemas.IntegrationsGithubReposRetrieveParams()
const IntegrationsGithubReposRetrieveQueryParams = orvalSchemas.IntegrationsGithubReposRetrieveQueryParams()
@@ -256,6 +304,8 @@ export const GENERATED_TOOLS: Record<string, () => ToolBase<ZodObjectAny>> = {
'integration-delete': integrationDelete,
'integration-get': integrationGet,
'integrations-channels-retrieve': integrationsChannelsRetrieve,
+ 'integrations-email-status-retrieve': integrationsEmailStatusRetrieve,
+ 'integrations-email-verify-create': integrationsEmailVerifyCreate,
'integrations-github-repos-retrieve': integrationsGithubReposRetrieve,
'integrations-jira-projects-retrieve': integrationsJiraProjectsRetrieve,
'integrations-linear-teams-retrieve': integrationsLinearTeamsRetrieve,
diff --git a/services/mcp/tests/unit/tool-filtering.test.ts b/services/mcp/tests/unit/tool-filtering.test.ts
index b8cef96c3655..a3f7be815200 100644
--- a/services/mcp/tests/unit/tool-filtering.test.ts
+++ b/services/mcp/tests/unit/tool-filtering.test.ts
@@ -1113,6 +1113,7 @@ describe('Tool Filtering - Feature Flags', () => {
'workflows-transactional-email-api',
'activity-summary',
'anomalies-search',
+ 'workflows-email-domain-agent-setup',
]
expect(allFlags).toEqual(expect.arrayContaining(branchFlags))
// The flags branches add are asserted on the line above and held out of the list and