Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

71.4 KB
Raw Blame
import { MessageHeader, SESv2Client, SendEmailCommand, SendEmailCommandInput } from '@aws-sdk/client-sesv2'
import { DateTime } from 'luxon'
import { SendMailOptions } from 'nodemailer'
import * as mimeFuncs from 'nodemailer/lib/mime-funcs'
import { Counter, Histogram } from 'prom-client'

import { CyclotronInvocationQueueParametersEmailType } from '~/cdp/schema/cyclotron'
import { HogFlowEmailSendingRateLimit, HogFlowEmailSendingRateLimitSchema } from '~/cdp/schema/hogflow'
import {
    CyclotronJobInvocationHogFunction,
    CyclotronJobInvocationResult,
    HogFunctionType,
    IntegrationType,
    MessageAssetRow,
} from '~/cdp/types'
import { createAddLogFunction, logEntry } from '~/cdp/utils'
import { createInvocationResult } from '~/cdp/utils/invocation-utils'
import { buildIntegerMatcher } from '~/common/config/matchers'
import { logger } from '~/common/utils/logger'

import { TRANSACTIONAL_EMAIL_EVENT } from '../hogflows/hogflow-sensitivity'
import type { DecodedAttachment } from '../hogflows/transactional-email'
import { IntegrationManagerService } from '../managers/integration-manager.service'
import { RecipientManagerRecipient, RecipientsManagerService } from '../managers/recipients-manager.service'
import { TeamWorkflowsConfigService } from '../managers/team-workflows-config.service'
import { WorkflowsActivationReporter } from '../monitoring/workflows-activation-reporter'
import { RateLimiterService } from '../rate-limiter/rate-limiter.service'
import {
    EMAIL_ATTACHMENTS_MISSING,
    EmailAttachmentsMissingError,
    EmailAttachmentsService,
} from './email-attachments.service'
import { getEmailQueuePriorityClass } from './email-priority'
import { selectEmailSenderIntegrationId } from './email-sender-selection'
import { EmailStatusReason, EmailStatusService, isTransactionalEmailSend } from './email-status.service'
import { EmailSuppressionService, SuppressionCheckOptions } from './email-suppression.service'
import {
    addTrackingToEmail,
    resolveEmailSendingVersion,
    resolveEmailTrackingDistinctId,
} from './email-tracking.service'
import { mailDevTransport, mailDevWebUrl } from './helpers/maildev'
import { maybeAddPreheaderToEmail } from './helpers/preheader'
import { SES_EMAIL_STATUS_TAG, SES_SENSITIVE_TAG } from './helpers/ses'
import { quoteMailbox, removeSuppressedRecipients, unquoteMailbox } from './helpers/suppressed-recipients'
import { EmailTrackingCodeSigner, TRACKING_CODE_HEADER_NAME } from './helpers/tracking-code'
import { addUtmTagsToEmail, renderUtmOverrides, resolveUtmTags } from './helpers/utm'
import { MessageAssetsService } from './message-assets.service'
import {
    PreferencesTokenSource,
    RecipientTokensService,
    preferencesTokenSourceForInvocation,
} from './recipient-tokens.service'

const sesThrottleResponsesTotal = new Counter({
    name: 'cdp_ses_throttle_responses_total',
    help: 'SES API responses classified as throttle/rate-limit. Sustained nonzero rate means the local bucket is set too high vs. the SES quota.',
    labelNames: ['error_code'],
})

/**
 * Every send on this path is generated by a workflow rather than typed by a person, so it
 * carries the RFC 3834 marker that says so. A well-behaved autoresponder does not answer a
 * message marked this way, which stops the loop that starts when a workflow emails an address
 * that answers automatically, such as a support inbox whose reply lands back in the same
 * workflow as a new ticket.
 *
 * `auto-generated` rather than `auto-replied`: the value covers the whole path, campaigns
 * included, not only the sends that answer an incoming message.
 */
const AUTO_SUBMITTED_HEADER_NAME = 'Auto-Submitted'
const AUTO_SUBMITTED_HEADER_VALUE = 'auto-generated'
const AUTO_SUBMITTED_HEADER: MessageHeader = { Name: AUTO_SUBMITTED_HEADER_NAME, Value: AUTO_SUBMITTED_HEADER_VALUE }

/**
 * SES error codes that signal a transient rate-limit shape — safe to retry
 * shortly after. `TooManyRequestsException` is the SES-v2-specific class;
 * `ThrottlingException` is a generic AWS SDK error code that can surface
 * from the underlying transport layer for the same condition (not exported
 * as a class for sesv2, so we match by `name`).
 *
 * `SendingPausedException` is *not* on this list — it signals a reputation
 * or account-state problem that won't recover in seconds. Retrying within
 * 500ms just burns reschedules; the job hard-fails instead, surfaces via
 * `email_failed`, and the underlying SES config needs operator attention.
 */
const SES_THROTTLE_ERROR_NAMES = ['TooManyRequestsException', 'ThrottlingException'] as const
type SesThrottleErrorName = (typeof SES_THROTTLE_ERROR_NAMES)[number]

function isSesThrottleError(error: unknown): error is Error & { name: SesThrottleErrorName } {
    return error instanceof Error && (SES_THROTTLE_ERROR_NAMES as readonly string[]).includes(error.name)
}

// Every other error name reads as `other`, so an unexpected SDK error cannot add a series.
const SES_SEND_ERROR_RESULTS = [
    'MessageRejected',
    'MailFromDomainNotVerifiedException',
    'AccountSuspendedException',
    'SendingPausedException',
    'LimitExceededException',
    'NotFoundException',
    'BadRequestException',
    'TimeoutError',
] as const
type SesSendResult = 'success' | 'throttled' | (typeof SES_SEND_ERROR_RESULTS)[number] | 'other'

function classifySesSendError(error: unknown): SesSendResult {
    if (isSesThrottleError(error)) {
        return 'throttled'
    }
    const name = error instanceof Error ? error.name : undefined
    return SES_SEND_ERROR_RESULTS.find((known) => known === name) ?? 'other'
}

const sesSendResultsTotal = new Counter({
    name: 'cdp_ses_send_results_total',
    help: 'SES SendEmail calls by result: success, throttled (rescheduled, with the error code in cdp_ses_throttle_responses_total), a known SES error name, or other.',
    labelNames: ['result'],
})

const sesSendDurationMs = new Histogram({
    name: 'cdp_ses_send_duration_ms',
    help: 'Duration of the SES SendEmail call in milliseconds, by whether SES accepted the email (success or error).',
    labelNames: ['result'],
    buckets: [25, 50, 100, 250, 500, 1_000, 2_500, 5_000, 10_000, 30_000],
})

const emailQueueToSendMs = new Histogram({
    name: 'cdp_email_queue_to_send_ms',
    help: 'Milliseconds an email waits on the email queue before its handoff to SES. For source="transactional_api" the wait starts when cdp-api created the job, so it is the time from the API\'s 202 to SES and includes every throttled or rate-limited retry. For source="workflow" the wait starts at the job\'s scheduled time: when the job joined the email queue, or when a delayed or rescheduled send became due, so a retried send measures only its last wait. priority_class is the email queue lane. team_capped is true when the team sending cap delayed the send, so that alerts can leave out a wait that the cap causes on purpose.',
    labelNames: ['source', 'priority_class', 'team_capped'],
    buckets: [100, 250, 500, 1_000, 2_500, 5_000, 10_000, 30_000, 60_000, 120_000, 300_000],
})

type EmailQueueWait = { source: 'transactional_api' | 'workflow'; startedAt: DateTime; teamCapped: boolean }

// A transactional API job is created on the email queue, so its creation time is when the caller got its 202. A
// workflow job can join the email queue long after it was created, so its wait starts at the time it became due.
// Both come from fields a clone keeps: running the email step clones the invocation and resets queueScheduledAt.
function emailQueueWait(invocation: CyclotronJobInvocationHogFunction): EmailQueueWait | undefined {
    const source =
        invocation.state.globals.event?.event === TRANSACTIONAL_EMAIL_EVENT ? 'transactional_api' : 'workflow'
    const startedAt = source === 'transactional_api' ? invocation.queueCreatedAt : invocation.queueDueAt
    return startedAt ? { source, startedAt, teamCapped: invocation.queueMetadata?.teamCapped === true } : undefined
}

/**
 * Tagged error signalling that SES rejected the send for a transient,
 * rate-limit-shaped reason. The caller schedules a retry instead of failing
 * the job. Carries the SES error name for metrics and the retry delay we
 * pick locally (SES doesn't return a Retry-After header).
 */
export class SESThrottleError extends Error {
    public readonly errorCode: SesThrottleErrorName
    public readonly retryAfterMs: number

    constructor(errorCode: SesThrottleErrorName, retryAfterMs: number, message: string) {
        super(message)
        this.name = 'SESThrottleError'
        this.errorCode = errorCode
        this.retryAfterMs = retryAfterMs
    }
}

class EmailAttachmentsUnavailableError extends Error {
    constructor(cause: unknown) {
        super('Could not load the attachment files of this email', { cause })
        this.name = 'EmailAttachmentsUnavailableError'
    }
}

function pickAttachmentLoadRetryDelayMs(tries: number): number {
    return Math.min(1_000 * tries + Math.floor(Math.random() * 1_000), 60_000)
}

class UnverifiedEmailDomainError extends Error {
    constructor() {
        super('The selected email integration domain is not verified')
        this.name = 'UnverifiedEmailDomainError'
    }
}

const isRealWorkflowSend = (invocation: CyclotronJobInvocationHogFunction, isTest: boolean): boolean =>
    !isTest && 'hogFlow' in invocation

function pickThrottleRetryDelayMs(): number {
    // Constant 500–1000ms jitter is plenty: the local Valkey bucket already
    // gates re-dequeue at the configured refill rate, so a quick retry will
    // simply re-claim a token if SES capacity has refreshed. Exponential
    // backoff isn't needed at this layer.
    return 500 + Math.floor(Math.random() * 500)
}

const workflowEmailRateLimitedTotal = new Counter({
    name: 'cdp_workflow_email_rate_limited_total',
    help: 'Email sends delayed by a user-configured per-workflow sending rate limit.',
})

// The metadata blob is the flow action's config plus flow-level keys stamped in by
// HogFlowFunctionsService.buildHogFunction; parse defensively since it is untyped.
function parseWorkflowEmailRateLimit(metadata: HogFunctionType['metadata']): HogFlowEmailSendingRateLimit | null {
    const raw = metadata?.email_sending_rate_limit
    if (!raw) {
        return null
    }
    const parsed = HogFlowEmailSendingRateLimitSchema.safeParse(raw)
    return parsed.success ? parsed.data : null
}

const workflowEmailPausedTotal = new Counter({
    name: 'cdp_workflow_email_paused_total',
    help: 'Email sends skipped because the workflow they belong to had its email paused for complaints or hard bounces.',
})

/**
 * Whether this workflow's email is paused, and the customer-facing reason.
 *
 * Stamped into the synthetic hog function's metadata by HogFlowFunctionsService, which is the only
 * flow-level state in scope here. The `HogFlow` post_save signal publishes a worker config reload
 * when the pause is written, so an in-flight run and an already-queued batch send both stop at this
 * choke point instead of only newly started runs.
 */
function parseWorkflowEmailPause(metadata: HogFunctionType['metadata']): { reason: string; byStaff: boolean } | null {
    if (!metadata?.email_sending_paused_at) {
        return null
    }
    const reason = metadata.email_sending_paused_reason
    return {
        reason: typeof reason === 'string' ? reason : '',
        byStaff: metadata.email_sending_paused_by === 'staff',
    }
}

function pickTokenBucketRetryDelayMs(refillPerSecond: number): number {
    // Wake around when the next token accrues. The 1x-2x jitter spreads a queued backlog's
    // retries so they don't all re-dequeue (and re-claim against one token) at the same instant.
    // Clamped so second-scale limits don't churn the queue and hour-scale limits still wake
    // often enough to drain promptly once capacity frees up.
    const tokenIntervalMs = 1000 / refillPerSecond
    const baseMs = Math.min(Math.max(tokenIntervalMs, 1_000), 5 * 60 * 1_000)
    return Math.floor(baseMs * (1 + Math.random()))
}

// Bounds for the non-reserved cap retry delays (fallbacks and horizon overflow; a
// reserved slot is parked on exactly). The floor keeps second-scale refills from
// churning the queue. The ceiling bounds how stale the computed wake time can get:
// capacity can appear earlier than computed (a limit raise, or an idle bucket
// expiring back to full capacity), and a parked job only notices when it wakes.
const CAP_RETRY_MIN_MS = 1_000
const CAP_RETRY_MAX_MS = 60 * 60 * 1_000

// How far denied sends park. Everything at or below the top bucket is a real slot.
// Above the top bucket is overflow: the backlog is deeper than one hour of refill.
// A sustained rate up there means a team queues more email than its limit can send.
const emailReservedParkMs = new Histogram({
    name: 'cdp_email_reserved_park_ms',
    help: 'How far into the future a rate-limit-denied email parked, by limiter.',
    labelNames: ['limiter'],
    buckets: [1_000, 5_000, 15_000, 60_000, 300_000, 900_000, 1_800_000, 3_600_000],
})

function pickReservedRetryDelayMs(retryAfterMs: number | null, refillPerSecond: number, reserved: boolean): number {
    // No horizon means the limiter itself failed, not that the bucket was empty.
    // Wake on the short token-bucket cadence, not on a cap that paces in hours.
    if (retryAfterMs === null) {
        return pickTokenBucketRetryDelayMs(refillPerSecond)
    }
    // A reserved slot is the caller's own, exactly one token interval behind the slot
    // in front. Park on it as-is, with no floor: waking off the slot in either
    // direction means the token is not there (the bucket banks no surplus), the send
    // gets denied again, and it goes to the back of the line.
    if (reserved) {
        return retryAfterMs
    }
    const parkMs = Math.max(retryAfterMs, CAP_RETRY_MIN_MS)
    // Past the horizon nothing is reserved: every overflow caller got this same wake
    // time back. Spread them over the next horizon so they do not arrive as one herd
    // asking for tokens that will not be there.
    return Math.floor(parkMs * (1 + Math.random()))
}

const teamEmailCapDelayedTotal = new Counter({
    name: 'cdp_team_email_cap_delayed_total',
    help: 'Workflow email sends delayed by the team trust-tier sending cap, or that would have been. `mode=shadow` and `mode=transactional` both mean the send went out anyway.',
    labelNames: ['tier', 'bucket', 'mode'],
})

/**
 * Rollout mode for the per-team sending cap.
 *  - `off`: the cap is not consulted at all.
 *  - `shadow`: the cap is evaluated and every send it would delay is counted and logged, but no
 *    send is delayed. This is how the tier distribution is checked against real traffic before
 *    anything is enforced.
 *  - `enforce`: a denied claim reschedules the send.
 */
export type TeamEmailCapMode = 'off' | 'shadow' | 'enforce'

/** An unrecognized mode reads as `off`, so a typo in the env cannot start throttling teams. */
export function parseTeamEmailCapMode(value: string | undefined): TeamEmailCapMode {
    const mode = (value ?? '').trim().toLowerCase()
    return mode === 'shadow' || mode === 'enforce' ? mode : 'off'
}

/** Parses a comma-separated cap table. Any unusable entry drops the whole table, and the send path
 * then applies DEFAULT_TEAM_EMAIL_TIER_DAILY_CAPS. A non-integer cap is unusable because the
 * all-or-nothing claim needs a whole-number request. */
export function parseTierCaps(value: string | undefined): number[] {
    const parts = (value ?? '')
        .split(',')
        .map((part) => part.trim())
        .filter((part) => part.length > 0)
    const caps = parts.map((part) => Number(part))
    if (caps.length === 0 || caps.some((cap) => !Number.isInteger(cap) || cap <= 0)) {
        return []
    }
    return caps
}

// Mirrors WORKFLOWS_EMAIL_TIER_DAILY_CAPS in posthog/settings/web.py. Both sides read the same
// tier index, so the two tables must stay in step.
const DEFAULT_TEAM_EMAIL_TIER_DAILY_CAPS = [100, 1000, 3000, 10000, 30000, 100000, 300000, 1000000]

// Comfortably longer than the window the bucket paces, so an idle bucket is never reset to full
// by expiry before it would have refilled on its own. Without that, a team that pauses for a day
// would get a fresh daily allowance instead of waiting for the refill.
const TEAM_EMAIL_DAY_BUCKET_TTL_SECONDS = 3 * 24 * 60 * 60

type TeamEmailCapBucket = {
    key: string
    capacity: number
    refillPerSecond: number
    ttlSeconds: number
    /** Cap and period as the customer sees them in the log line. */
    label: string
}

export function teamEmailCapBucket(teamId: number, dailyCap: number): TeamEmailCapBucket {
    return {
        key: `@posthog/team-email-rate-day/{${teamId}}`,
        capacity: dailyCap,
        refillPerSecond: dailyCap / 86400,
        ttlSeconds: TEAM_EMAIL_DAY_BUCKET_TTL_SECONDS,
        label: `${dailyCap.toLocaleString('en-US')} emails per day`,
    }
}

export interface EmailServiceConfig {
    sesAccessKeyId: string
    sesSecretAccessKey: string
    sesRegion: string
    sesEndpoint: string
    // Configuration set with ESP-level open/click tracking enabled.
    sesTrackedConfigurationSet: string
    // Configuration set without open/click tracking. Empty means not provisioned: tracking-off
    // sends fall back to the tracked set (with a warning) rather than failing.
    sesUntrackedConfigurationSet: string
    // Trust-tiered per-team sending caps. Optional and defaulting to off, so every send path that
    // builds an EmailService without them keeps its pre-cap behavior.
    teamEmailCapMode?: TeamEmailCapMode
    teamEmailTierDailyCaps?: number[]
    utmLinkScannerTeams?: string
}

/**
 * Strips control characters from an email subject to prevent header injection
 * and delivery issues. Removes ASCII 0-31 (except horizontal tab) and DEL (127).
 */
export function sanitizeEmailSubject(subject: string): string {
    return subject
        .replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, '')
        .replace(/[\r\n]+/g, ' ')
        .trim()
}

// Splits a comma-separated address list and extracts the bare email from any RFC-822
// `"Name" <email@x>` entries. Used by the pre-send suppression check to normalize cc/bcc entries
// before matching against the suppression list (which stores bare, lower-cased addresses).
export function extractEmailsFromAddressList(value: string | undefined): string[] {
    if (typeof value !== 'string' || value.trim().length === 0) {
        return []
    }
    return value
        .split(',')
        .map((raw) => {
            const trimmed = raw.trim()
            const bracketed = trimmed.match(/<([^>]+)>/)
            return (bracketed ? bracketed[1] : trimmed).trim()
        })
        .filter((addr) => addr.length > 0)
}

// Deliberately stricter than RFC 5322: exactly one @, a dotted domain, and none of the
// characters that would let a templated value smuggle a second address or break out of the
// RFC-822 `"Name" <email>` framing (whitespace, quotes, angle brackets, list separators).
const FROM_OVERRIDE_EMAIL_REGEX = /^[^\s@"<>,;]+@[^\s@"<>,;]+\.[^\s@"<>,;]+$/
const MAX_FROM_OVERRIDE_EMAIL_LENGTH = 254

const PRINTABLE_ASCII = /^[\x20-\x7E]*$/
const RFC_2047_MAX_ENCODED_WORD_LENGTH = 75

function cleanDisplayName(name: string | undefined): string {
    return (name ?? '').replace(/[\x00-\x1F\x7F]/g, '').trim()
}

function encodeDisplayName(name: string): string {
    return PRINTABLE_ASCII.test(name)
        ? quoteDisplayName(name)
        : mimeFuncs.encodeWord(name, 'Q', RFC_2047_MAX_ENCODED_WORD_LENGTH)
}

function quoteDisplayName(name: string): string {
    return `"${name.replace(/[\\"]/g, '\\$&')}"`
}

export function formatMailbox(name: string | undefined, email: string): string {
    const displayName = cleanDisplayName(name)
    return displayName ? `${encodeDisplayName(displayName)} <${email}>` : email
}

function toNodemailerAddress(name: string | undefined, email: string): { name: string; address: string } {
    return { name: cleanDisplayName(name), address: email }
}

export function parseAddressList(value?: string): string[] | undefined {
    if (!value || !value.trim()) {
        return undefined
    }
    const result = value
        .split(',')
        .map((addr) => addr.trim())
        .filter((addr) => addr.length > 0)
    return result.length > 0 ? result : undefined
}

export class EmailService {
    sesV2Client: SESv2Client | null

    private recipientTokensService: RecipientTokensService
    private untrackedConfigSetWarningLogged = false

    constructor(
        private sesConfig: EmailServiceConfig,
        private integrationManager: IntegrationManagerService,
        private teamWorkflowsConfigService: TeamWorkflowsConfigService,
        encryptionSaltKeys: string,
        private siteUrl: string,
        private trackingCodeSigner: EmailTrackingCodeSigner,
        private emailSuppressionService: EmailSuppressionService,
        private recipientsManager: RecipientsManagerService,
        // Required, so that a caller which builds the service without it fails to compile. Pass null
        // only where no email status is recorded.
        private emailStatusService: EmailStatusService | null,
        private workflowsActivationReporter: Pick<WorkflowsActivationReporter, 'report'>,
        private messageAssetsService?: MessageAssetsService,
        private workflowEmailRateLimiter: RateLimiterService | null = null,
        private teamEmailRateLimiter: RateLimiterService | null = null,
        private emailAttachmentsService: EmailAttachmentsService | null = null
    ) {
        this.sesV2Client = this.sesConfig.sesRegion
            ? new SESv2Client({
                  region: this.sesConfig.sesRegion,
                  endpoint: this.sesConfig.sesEndpoint || undefined,
              })
            : null
        this.recipientTokensService = new RecipientTokensService(encryptionSaltKeys, siteUrl)
    }

    // Send email. `isTest` flags sends from the editor's "Run test" path so the tracking code
    // embedded in the email tells the SES webhook to skip recording metrics for test traffic.
    public async executeSendEmail(
        invocation: CyclotronJobInvocationHogFunction,
        isTest = false
    ): Promise<CyclotronJobInvocationResult<CyclotronJobInvocationHogFunction>> {
        if (invocation.queueParameters?.type !== 'email') {
            throw new Error('Invocation passed to sendEmail is not an email function')
        }

        const result = createInvocationResult<CyclotronJobInvocationHogFunction>(
            invocation,
            // Preserve the incoming priority: createInvocationResult otherwise resets it to 0, which
            // on a throttle reschedule (below) would rewrite the send's priority class — an entering
            // bulk send (priority 1) would return as fast-lane (0). The queue caller sets this to the
            // send's class before calling in, so carrying it through keeps a throttled retry in class.
            { queuePriority: invocation.queuePriority },
            {
                finished: true,
            }
        )
        const addLog = createAddLogFunction(result.logs)

        // Read from the incoming invocation: result.invocation is a clone, and a clone drops the scheduled time.
        const queueWait = invocation.queue === 'email' && !isTest ? emailQueueWait(invocation) : undefined

        const params = invocation.queueParameters
        const integrationId = selectEmailSenderIntegrationId(invocation.id, params.from)
        const integration = await this.integrationManager.get(integrationId)

        let success: boolean = false
        let rescheduled: boolean = false
        let assetRow: MessageAssetRow | null = null
        let trackingEnabled = true
        let sendingDomain = ''
        let failureReason: EmailStatusReason = 'provider_error'

        try {
            // Team-level kill switches: staff suspend all workflow email for a team whose sender
            // reputation endangers shared SES deliverability, and AWS pauses the team's SES tenant
            // for the same underlying reason. Same choke-point placement as the suppression check
            // below so no upstream route can bypass either. Test sends are blocked too — they hit
            // SES and count against the tenant all the same. A paused tenant rejects every send, so
            // gating here turns wasted invocations into one clear skip.
            const suspensionCause = await this.teamWorkflowsConfigService.getEmailSendingSuspension(invocation.teamId)
            if (suspensionCause) {
                const skipReason =
                    suspensionCause === 'staff'
                        ? 'Skipping send: email sending is suspended for this project. Contact support to get sending re-enabled.'
                        : 'Skipping send: our email provider paused sending for this project. Check the Reputation tab to see what to fix.'
                addLog('warn', skipReason)
                result.emailSendResult = { status: 'skipped', reason: skipReason }
                this.emailStatusService?.queueSkippedSend(invocation, 'sending_suspended', isTest)
                logger.warn('Skipping email send for a suspended team', {
                    teamId: invocation.teamId,
                    cause: suspensionCause,
                })
                // One metric for both causes: from the project's side sending is off either way, so
                // the cause belongs in the log lines above and not in a second metric name.
                if (!isTest) {
                    result.metrics.push({
                        team_id: invocation.teamId,
                        app_source_id: invocation.parentRunId ?? invocation.functionId,
                        instance_id: invocation.state.actionId || invocation.id,
                        metric_kind: 'email',
                        metric_name: 'email_suspended',
                        count: 1,
                    })
                }
                result.invocation.state.vmState?.stack.push({ success: false })
                return result
            }

            // Per-workflow pause: this workflow's complaint or hard bounce rate breached a
            // threshold, so its email is held while the rest of the project keeps sending. Placed
            // at the same choke point as the team-level switch above, for the same reason. Test
            // sends are blocked too, because they hit SES and count against the tenant all the same.
            const workflowPause = parseWorkflowEmailPause(invocation.hogFunction.metadata)
            if (workflowPause) {
                workflowEmailPausedTotal.inc()
                const pauseDetail = workflowPause.reason ? ` ${workflowPause.reason}` : ''
                // Error rather than warn: the email did not go out, and warn-level lines get skimmed
                // past in the run logs. The run itself still continues; a pause is policy, not a
                // fault, so it must not enter retry or abort handling.
                const skipReason = workflowPause.byStaff
                    ? `Skipping send: PostHog staff paused email sending for this workflow.${pauseDetail} Contact support to get sending re-enabled.`
                    : `Skipping send: email sending is paused for this workflow.${pauseDetail} Resume it from the workflow page once the audience is cleaned up.`
                addLog('error', skipReason)
                result.emailSendResult = { status: 'skipped', reason: skipReason }
                this.emailStatusService?.queueSkippedSend(invocation, 'sending_paused', isTest)
                if (!isTest) {
                    result.metrics.push({
                        team_id: invocation.teamId,
                        app_source_id: invocation.parentRunId ?? invocation.functionId,
                        instance_id: invocation.state.actionId || invocation.id,
                        metric_kind: 'email',
                        metric_name: 'email_paused',
                        count: 1,
                    })
                }
                // Mark the skip so the flow-level billing gate does not charge for a send that
                // never reached the provider, and does not re-pin the run's attribution version to
                // one that delivered nothing. Mirrors the pre-send opt-out and bounce skip paths.
                result.skipped = true
                result.invocation.state.vmState?.stack.push({ success: false })
                return result
            }

            // Wrong-team references deliberately read as not-found so an ID's existence on another team can't be probed
            if (!integration || integration.team_id !== invocation.teamId) {
                throw new Error(
                    "Email integration not found. The sender configured for this step no longer exists — select a new sender in the workflow's email step."
                )
            }
            if (integration.kind !== 'email') {
                throw new Error(
                    "The integration configured for this step is not an email channel — select an email sender in the workflow's email step."
                )
            }

            const from = this.resolveFromSender(integration, params.from, addLog)
            sendingDomain = from.email.split('@').pop()?.toLowerCase() ?? ''

            // An empty `to` comes from the customer's data (a person without an email property, or no
            // person at all), not from a fault in the send path. SES rejects it as an invalid address,
            // so record a skip with its own metric instead of an `email_failed`. Otherwise one project
            // with bad data inflates the platform send failure rate.
            if (!params.to?.email?.trim()) {
                const skipReason =
                    'Skipping send: the recipient has no email address. Add a filter so only people with an email address reach this step.'
                addLog('warn', skipReason)
                result.emailSendResult = { status: 'skipped', reason: skipReason }
                if (!isTest) {
                    result.metrics.push({
                        team_id: invocation.teamId,
                        app_source_id: invocation.parentRunId ?? invocation.functionId,
                        instance_id: invocation.state.actionId || invocation.id,
                        metric_kind: 'email',
                        metric_name: 'email_missing_recipient',
                        count: 1,
                    })
                }
                this.emailStatusService?.queueSkippedSend(invocation, 'missing_recipient', isTest)
                if (isRealWorkflowSend(invocation, isTest)) {
                    void this.workflowsActivationReporter.report(invocation.teamId, 'workflows send blocked', {
                        reason: 'missing_recipient',
                        channel: 'email',
                        workflow_id: invocation.functionId,
                    })
                }
                result.skipped = true
                result.invocation.state.vmState?.stack.push({ success: false })
                return result
            }

            // Single choke point for the suppression check — every send path lands here regardless
            // of whether the invocation came from a workflow action or an email destination hog
            // function. Checking here means callers can't bypass it by taking a different upstream
            // route. Suppressed addresses are removed from every list. A suppressed `to` is replaced by
            // the first deliverable cc address, and the send is skipped only when none is left.
            const suppressionOptions: SuppressionCheckOptions = {
                transactional: invocation.hogFunction?.metadata?.message_category_type === 'transactional',
            }
            const suppression = await removeSuppressedRecipients(params, (email) =>
                this.emailSuppressionService.isSuppressed(invocation.teamId, email, suppressionOptions)
            )
            if (suppression.skip) {
                const skipReason = `Skipping send: ${suppression.removed.join(', ')} ${suppression.removed.length === 1 ? 'is' : 'are'} on the suppression list and no cc address can receive the email instead`
                addLog('info', skipReason)
                result.emailSendResult = { status: 'skipped', reason: skipReason }
                this.emailStatusService?.queueSkippedSend(invocation, 'recipient_suppressed', isTest)
                if (!isTest) {
                    result.metrics.push({
                        team_id: invocation.teamId,
                        app_source_id: invocation.parentRunId ?? invocation.functionId,
                        instance_id: invocation.state.actionId || invocation.id,
                        metric_kind: 'email',
                        metric_name: 'email_suppressed',
                        count: 1,
                    })
                }
                result.invocation.state.vmState?.stack.push({ success: false })
                return result
            }
            if (suppression.removed.length > 0) {
                const promoted = suppression.promotedTo
                    ? ` ${suppression.promotedTo.email} moves from cc to the To field.`
                    : ''
                addLog(
                    'warn',
                    `Not sending to ${suppression.removed.join(', ')} because ${suppression.removed.length === 1 ? 'it is' : 'they are'} on the suppression list.${promoted} The email still goes to the other recipients.`
                )
            }
            // Always send the lists the suppression check parsed, so delivery uses the addresses it checked.
            const deliverableParams = {
                ...params,
                to:
                    suppression.promotedTo ??
                    (params.to?.email
                        ? { ...params.to, email: quoteMailbox(unquoteMailbox(params.to.email)) }
                        : params.to),
                cc: suppression.cc,
                bcc: suppression.bcc,
            }

            // Like suppression, the tracking decision lives at this choke point so every send path
            // (workflow action or email destination hog function) resolves it the same way.
            trackingEnabled = await this.resolveTrackingEnabled(result.invocation, deliverableParams)

            // User-configured per-workflow pacing. Claimed last, after every skip gate, so a
            // suspended or suppressed send never spends a token. Test sends bypass it. When the
            // limiter's Valkey is down claimUpTo returns 0, so sends wait (never drop) until it
            // recovers — same fail-closed stance as the global SES gate.
            const workflowRateLimit = parseWorkflowEmailRateLimit(invocation.hogFunction.metadata)
            if (workflowRateLimit && this.workflowEmailRateLimiter && !isTest) {
                const periodSeconds = workflowRateLimit.period === 'minute' ? 60 : 3600
                const refillPerSecond = workflowRateLimit.count / periodSeconds
                // Burst capacity is about one second of budget, not the full count: a bucket that
                // could hold `count` starts full and refills within the same period, so a fresh (or
                // idle-expired) bucket would send ~2x the configured limit in its first period.
                // A near-empty bucket keeps every window at ~count and spreads sends evenly, which
                // is what the pacing is for.
                const capacity = Math.max(1, Math.ceil(refillPerSecond))
                const claim = await this.workflowEmailRateLimiter.claimOrReserve(
                    {
                        key: `@posthog/workflow-email-rate/${invocation.teamId}/${invocation.functionId}`,
                        requested: 1,
                        capacity,
                        refillPerSecond,
                    },
                    CAP_RETRY_MAX_MS
                )
                if (claim.granted === 0) {
                    workflowEmailRateLimitedTotal.inc()
                    result.finished = false
                    // Re-attach the email payload before rescheduling. createInvocationResult cleared
                    // queueParameters, so without this the rescheduled dequeue has no 'email' params to
                    // re-enter the send path — the retry would resume the Hog VM instead and drop the
                    // send. Mirrors the fetch-retry (`result.invocation.queueParameters = params`) and
                    // queue-routing paths, which re-attach the same way.
                    result.invocation.queueParameters = params
                    result.invocation.queueMetadata = invocation.queueMetadata
                    const retryDelayMs = pickReservedRetryDelayMs(claim.retryAfterMs, refillPerSecond, claim.reserved)
                    emailReservedParkMs.labels('workflow-email').observe(retryDelayMs)
                    result.invocation.queueScheduledAt = DateTime.utc().plus({ milliseconds: retryDelayMs })
                    addLog(
                        'info',
                        `Sending rate limit reached (${workflowRateLimit.count} emails per ${workflowRateLimit.period}); retrying this email in ${Math.round(retryDelayMs / 1000)}s`
                    )
                    return result
                }
            }

            // Project-wide pacing from the team's trust tier. Sits next to the per-workflow limit
            // above and behaves the same way: it delays, never drops, and test sends bypass it.
            // Transactional sends bypass the delay too, and are counted instead.
            // Charged per recipient, not per send: SES counts every to/cc/bcc address against its
            // own quota, so a send with many copies must spend that many tokens.
            const capRecipients =
                1 +
                extractEmailsFromAddressList(deliverableParams.cc).length +
                extractEmailsFromAddressList(deliverableParams.bcc).length
            const capDelay = await this.claimTeamSendingBudget(invocation, isTest, capRecipients)
            if (capDelay) {
                result.finished = false
                // Re-attach the email payload before rescheduling, for the same reason as the
                // per-workflow limit above: createInvocationResult cleared queueParameters, and
                // without them the rescheduled dequeue resumes the Hog VM and drops the send.
                result.invocation.queueParameters = params
                result.invocation.queueMetadata = { ...invocation.queueMetadata, teamCapped: true }
                result.invocation.queueScheduledAt = DateTime.utc().plus({ milliseconds: capDelay.retryDelayMs })
                const capRetrySeconds = Math.round(capDelay.retryDelayMs / 1000)
                addLog(
                    'info',
                    capDelay.label
                        ? `This project reached its email sending limit of ${capDelay.label}. Retrying this email in ${capRetrySeconds}s. The limit rises as the project builds a clean sending history.`
                        : `Could not check this project's email sending limit, so this email is waiting. Retrying in ${capRetrySeconds}s. No action is needed on your side.`
                )
                return result
            }

            // A reply-to set on the email step wins over the sender's default.
            const paramsWithReplyTo =
                parseAddressList(deliverableParams.replyTo) || !integration.config.reply_to
                    ? deliverableParams
                    : { ...deliverableParams, replyTo: integration.config.reply_to }

            // Tagged before click tracking wraps the links, so the tags land on the destination URL.
            const metadata = invocation.hogFunction.metadata
            const sendParams =
                metadata?.utm_tags_enabled === true && params.html
                    ? {
                          ...paramsWithReplyTo,
                          html: addUtmTagsToEmail(
                              params.html,
                              resolveUtmTags(
                                  {
                                      utm_source: 'posthog',
                                      utm_medium: 'email',
                                      utm_campaign: metadata.hog_flow_name ?? invocation.hogFunction.name,
                                      utm_content: metadata.hog_flow_action_name ?? '',
                                  },
                                  renderUtmOverrides(metadata.utm_params, invocation.state.globals, (key, message) =>
                                      addLog(
                                          'warn',
                                          `Used the default ${key} because its value has an error: ${message}`
                                      )
                                  )
                              ),
                              this.siteUrl,
                              buildIntegerMatcher(this.sesConfig.utmLinkScannerTeams, true)(invocation.teamId)
                          ),
                      }
                    : paramsWithReplyTo

            const files = await this.loadAttachments(invocation.id, invocation.teamId, params.attachments)

            switch (integration.config.provider ?? 'ses') {
                case 'maildev':
                    await this.sendEmailWithMaildev(result, sendParams, from, trackingEnabled, isTest, files)
                    break
                case 'ses':
                    await this.sendEmailWithSES(result, sendParams, from, trackingEnabled, isTest, queueWait, files)
                    break

                case 'unsupported':
                    throw new Error('Email delivery mode not supported')
            }

            // Emit the `[Email:…]` token in the success log only when an asset row
            // will actually be captured; `renderWorkflowLogMessage` renders it as the
            // "View email" chip, so suppressing it for skipped captures keeps the chip
            // from 404-ing on click.
            if (!isTest && this.messageAssetsService) {
                assetRow = this.messageAssetsService.buildRowForEmail(invocation, sendParams)
            }
            const viewEmailToken = assetRow ? ` [Email:${invocation.id}:${invocation.state.actionId ?? ''}]` : ''
            addLog('info', `Email sent to ${sendParams.to.email} from ${from.name} <${from.email}>${viewEmailToken}`)
            success = true
            result.emailSendResult = { status: 'accepted' }
        } catch (error) {
            if (error instanceof SESThrottleError) {
                // Treat as a transient delivery delay — reschedule rather than fail
                // the job. Our local bucket is the primary throttle; this path
                // catches the cases where SES disagrees with our estimate.
                rescheduled = true
                result.finished = false
                result.invocation.queueParameters = params
                result.invocation.queueMetadata = invocation.queueMetadata
                result.invocation.queueScheduledAt = DateTime.utc().plus({ milliseconds: error.retryAfterMs })
                addLog('warn', `SES rate-limited (${error.errorCode}); rescheduling email in ${error.retryAfterMs}ms`)
            } else if (error instanceof EmailAttachmentsUnavailableError) {
                const tries = (invocation.queueMetadata?.attachmentLoadTries ?? 0) + 1
                const retryDelayMs = pickAttachmentLoadRetryDelayMs(tries)
                rescheduled = true
                result.finished = false
                result.invocation.queueParameters = params
                result.invocation.queueMetadata = { ...invocation.queueMetadata, attachmentLoadTries: tries }
                result.invocation.queueScheduledAt = DateTime.utc().plus({ milliseconds: retryDelayMs })
                addLog('warn', `${error.message}; rescheduling email in ${retryDelayMs}ms`)
                logger.warn('⚠️', 'Failed to load email attachment rows', {
                    job_id: invocation.id,
                    tries,
                    error: error.cause instanceof Error ? error.cause.message : String(error.cause),
                })
            } else {
                if (error instanceof EmailAttachmentsMissingError) {
                    failureReason = EMAIL_ATTACHMENTS_MISSING
                }
                addLog('error', error.message)
                result.error = error.message
                result.emailSendResult = { status: 'failed', reason: error.message }
                result.finished = true
                if (error instanceof UnverifiedEmailDomainError && isRealWorkflowSend(invocation, isTest)) {
                    void this.workflowsActivationReporter.report(invocation.teamId, 'workflows send failed', {
                        reason: 'unverified_domain',
                        channel: 'email',
                        workflow_id: invocation.functionId,
                    })
                }
            }
        }

        if (rescheduled) {
            // On throttle, skip both the VM-state push and the business-metric
            // emit. The eventual successful retry will produce `email_sent` and
            // push the success bit to the VM stack — pushing them now would
            // double-count and lie about the send outcome.
            return result
        }

        // Push the response to the VM stack if running inline (not from the email queue)
        result.invocation.state.vmState?.stack.push({
            success,
        })

        // Test sends (from the editor's "Run test") must not record metrics — keep them out of the
        // workflow's Metrics tab, mirroring the isTest skip the SES webhook applies to delivery/open/click.
        if (!isTest) {
            result.metrics.push({
                team_id: invocation.teamId,
                app_source_id: invocation.parentRunId ?? invocation.functionId,
                instance_id: invocation.state.actionId || invocation.id,
                metric_kind: 'email',
                metric_name: success ? 'email_sent' : 'email_failed',
                count: 1,
            })

            // Untracked sends can never produce opens/clicks, so record them separately: open/click
            // rates are computed against (delivered - untracked) to avoid deflation.
            if (success && !trackingEnabled) {
                result.metrics.push({
                    team_id: invocation.teamId,
                    app_source_id: invocation.parentRunId ?? invocation.functionId,
                    instance_id: invocation.state.actionId || invocation.id,
                    metric_kind: 'email',
                    metric_name: 'email_untracked',
                    count: 1,
                })
            }

            if (success && assetRow) {
                result.messageAssets.push(assetRow)
            }

            // A failure here is final: SES throttling returned above to reschedule, and nothing
            // retries a failed send.
            if (this.emailStatusService && isTransactionalEmailSend(invocation)) {
                this.emailStatusService.queue({
                    teamId: invocation.teamId,
                    messageId: invocation.id,
                    status: success ? 'sent' : 'failed',
                    reason: success ? undefined : failureReason,
                    sendingDomain,
                })
            }
        }

        const distinctId = resolveEmailTrackingDistinctId(invocation)
        if (
            distinctId &&
            !isTest &&
            (await this.teamWorkflowsConfigService.shouldCaptureEngagementEvents(invocation.teamId))
        ) {
            result.capturedPostHogEvents.push({
                team_id: invocation.teamId,
                timestamp: new Date().toISOString(),
                distinct_id: distinctId,
                event: success ? '$workflows_email_sent' : '$workflows_email_failed',
                properties: {
                    $workflow_id: invocation.functionId,
                    $workflow_action_id: invocation.state.actionId,
                    $email_to: params.to.email,
                    // A sensitive template's subject can carry a login code, so its events leave it out.
                    ...(invocation.hogFunction?.metadata?.sensitive === true ? {} : { $email_subject: params.subject }),
                    // Always set, never conditional: an untracked send can never produce a
                    // `$workflows_email_opened` or `$workflows_email_link_clicked`, so without this
                    // dimension on the send there is no way to build an open rate in an insight that
                    // isn't deflated by however much of the audience declined tracking.
                    $email_tracking_enabled: trackingEnabled,
                },
            })
        }

        return result
    }

    /**
     * Claims one send (as `recipients` tokens, one per delivered copy) against the team's
     * trust-tier daily bucket.
     *
     * Returns the delay to reschedule with when the cap is reached, or null when the send may go
     * out. The cap bounds how much damage a team can do to the shared SES account's reputation
     * in a day. It does not spread that volume out: a full bucket lets the team send its whole
     * daily cap as one burst, and only volume past that waits for the refill.
     *
     * Failure stances differ on purpose. A failed tier lookup lets the send through, because a
     * config blip must never throttle a legitimate customer. A failed bucket claim delays the
     * send, because we must not send when we cannot account for it. That second case returns a
     * null `label`: the send waits, but no cap was reached, so the caller must not tell the
     * customer it hit one.
     *
     * Transactional sends are charged but never delayed, because a password reset that arrives
     * two hours late has already failed the person waiting on it. The charge does not bound the
     * exemption: claimUpTo records no debt, so once the bucket empties further transactional
     * sends cost nothing. No other control bounds that volume per team either. The SES limiter
     * paces the whole account, so what is left is suspension and the daily tier job, which
     * demotes a team on its complaint and bounce rates after the fact.
     */
    private async claimTeamSendingBudget(
        invocation: CyclotronJobInvocationHogFunction,
        isTest: boolean,
        recipients: number = 1
    ): Promise<{ retryDelayMs: number; label: string | null } | null> {
        const mode: TeamEmailCapMode = this.sesConfig.teamEmailCapMode ?? 'off'
        if (mode === 'off' || isTest || !this.teamEmailRateLimiter) {
            return null
        }

        const teamTier = await this.teamWorkflowsConfigService.getEmailSendingTier(invocation.teamId)
        if (teamTier === null) {
            return null
        }

        const dailyCaps = this.sesConfig.teamEmailTierDailyCaps?.length
            ? this.sesConfig.teamEmailTierDailyCaps
            : DEFAULT_TEAM_EMAIL_TIER_DAILY_CAPS
        const tier = Math.min(Math.max(teamTier, 0), dailyCaps.length - 1)

        const bucket = teamEmailCapBucket(invocation.teamId, dailyCaps[tier])
        // Clamped to the capacity: a send with more copies than a whole allowance could never be
        // granted and would reschedule forever. It charges the entire bucket instead, so it still
        // waits for full capacity and pays everything there is.
        const requested = Math.min(Math.max(1, recipients), bucket.capacity)

        const isTransactional = invocation.hogFunction?.metadata?.message_category_type === 'transactional'

        if (mode === 'enforce' && !isTransactional) {
            const claim = await this.teamEmailRateLimiter.claimAllOrNothing({ ...bucket, requested }, CAP_RETRY_MAX_MS)
            if (claim.granted) {
                return null
            }
            const capReached = claim.retryAfterMs !== null
            teamEmailCapDelayedTotal.inc({
                tier: String(tier),
                bucket: capReached ? 'day' : 'limiter_unavailable',
                mode,
            })
            const retryDelayMs = pickReservedRetryDelayMs(claim.retryAfterMs, bucket.refillPerSecond, claim.reserved)
            emailReservedParkMs.labels('team-email').observe(retryDelayMs)
            return {
                retryDelayMs,
                label: capReached ? bucket.label : null,
            }
        }

        // Shadow mode, and transactional sends in any mode: the send goes out regardless, so
        // drain the bucket and log the cap. Separate labels keep shadow a forecast of enforce.
        const observedMode = isTransactional ? 'transactional' : mode
        const granted = await this.teamEmailRateLimiter.claimUpTo({ ...bucket, requested })
        if (granted < requested) {
            teamEmailCapDelayedTotal.inc({ tier: String(tier), bucket: 'day', mode: observedMode })
            logger.info(
                '📧',
                isTransactional
                    ? 'Transactional email sent past the team sending cap'
                    : 'Team email sending cap would have delayed this send',
                {
                    teamId: invocation.teamId,
                    tier,
                    bucket: 'day',
                    cap: bucket.label,
                }
            )
        }
        return null
    }

    // Per-send tracking decision, combining two independent controls:
    // 1. Message-level: `tracking_enabled` on the email step's config (via `hogFunction.metadata`,
    //    same as `message_category_type`). False is a hard off regardless of consent. Absent means
    //    tracked, so email destination hog functions (which have no such config) keep tracking.
    // 2. Recipient-level consent (marketing only; transactional exempt per CNIL): the team's
    //    consent mode combined with the recipient's stored $email_tracking preference.
    // Consent lookup failures resolve to untracked (fail closed): tracking without verifiable
    // consent is a compliance risk, while an untracked send only costs metrics.
    private async resolveTrackingEnabled(
        invocation: CyclotronJobInvocationHogFunction,
        params: CyclotronInvocationQueueParametersEmailType
    ): Promise<boolean> {
        if (invocation.hogFunction?.metadata?.tracking_enabled === false) {
            return false
        }
        if (invocation.hogFunction?.metadata?.message_category_type === 'transactional') {
            return true
        }

        try {
            const consentMode = await this.teamWorkflowsConfigService.getEmailTrackingConsentMode(invocation.teamId)
            if (consentMode === 'off') {
                return true
            }

            // The pixel and rewritten links are whole-message artifacts delivered to every list,
            // so consent must hold for every recipient (same reasoning as the suppression check).
            const recipients: string[] = []
            if (params.to?.email && params.to.email.trim()) {
                recipients.push(unquoteMailbox(params.to.email))
            }
            // Preferences are stored against bare addresses, so a quoted local part is looked up unquoted.
            recipients.push(...extractEmailsFromAddressList(params.cc).map(unquoteMailbox))
            recipients.push(...extractEmailsFromAddressList(params.bcc).map(unquoteMailbox))

            const consents = await Promise.all(
                recipients.map(async (email) => {
                    const recipient = await this.recipientsManager.get({
                        teamId: invocation.teamId,
                        identifier: email,
                    })
                    return recipient ? this.recipientsManager.getEmailTrackingPreference(recipient) : 'NO_PREFERENCE'
                })
            )
            return consents.every((consent) =>
                consentMode === 'opt_in' ? consent === 'OPTED_IN' : consent !== 'OPTED_OUT'
            )
        } catch (error) {
            logger.warn('Email tracking consent lookup failed - sending untracked', {
                teamId: invocation.teamId,
                functionId: invocation.functionId,
                error: error instanceof Error ? error.message : String(error),
            })
            return false
        }
    }

    // The tracked and untracked configuration sets share the same delivery/bounce/complaint event
    // destination (suppression and the Metrics tab depend on those events); they differ only in
    // ESP-level open/click tracking.
    private resolveConfigurationSetName(
        trackingEnabled: boolean,
        invocation: CyclotronJobInvocationHogFunction
    ): string {
        if (trackingEnabled) {
            return this.sesConfig.sesTrackedConfigurationSet
        }
        if (this.sesConfig.sesUntrackedConfigurationSet) {
            return this.sesConfig.sesUntrackedConfigurationSet
        }
        // The missing set is static per process - one warning is signal, one per send is noise.
        if (!this.untrackedConfigSetWarningLogged) {
            this.untrackedConfigSetWarningLogged = true
            logger.warn(
                'Email tracking disabled for send but no untracked SES configuration set is configured - falling back to the tracked set, ESP-level open/click tracking may still apply',
                { teamId: invocation.teamId, functionId: invocation.functionId }
            )
        }
        return this.sesConfig.sesTrackedConfigurationSet
    }

    private resolveFromSender(
        integration: IntegrationType,
        from: CyclotronInvocationQueueParametersEmailType['from'],
        addLog: ReturnType<typeof createAddLogFunction>
    ): { email: string; name: string } {
        if (!integration.config.verified) {
            throw new UnverifiedEmailDomainError()
        }

        if (!integration.config.email || !integration.config.name) {
            throw new Error('The selected email integration is not configured correctly')
        }

        // Overrides arrive already rendered by the templating engine, so a template that
        // resolved to nothing means "no override" and the integration's stored sender applies.
        const overrideName = cleanDisplayName(from.name)

        return {
            email: this.resolveFromEmailAddress(integration, from.email?.trim(), addLog),
            name: overrideName || integration.config.name,
        }
    }

    // An unusable override degrades to the integration's own sender rather than failing the send.
    // Steps authored before mid-2026 carry a placeholder address written by an old sender picker,
    // so throwing here fails sends whose author never typed an address at all.
    private resolveFromEmailAddress(
        integration: IntegrationType,
        overrideEmail: string | undefined,
        addLog: ReturnType<typeof createAddLogFunction>
    ): string {
        if (!overrideEmail) {
            return integration.config.email
        }

        if (overrideEmail.length > MAX_FROM_OVERRIDE_EMAIL_LENGTH) {
            addLog(
                'warn',
                `Ignoring the custom sender address because it is too long. Sending from ${integration.config.email} instead. Use a From address with ${MAX_FROM_OVERRIDE_EMAIL_LENGTH} characters or fewer.`
            )
            return integration.config.email
        }

        if (!FROM_OVERRIDE_EMAIL_REGEX.test(overrideEmail)) {
            addLog(
                'warn',
                `Ignoring the custom sender address "${overrideEmail}": it is not a valid email address. Sending from ${integration.config.email} instead. Fix the From address in the workflow's email step so it resolves to a single valid address.`
            )
            return integration.config.email
        }

        // Verification is domain-level (the DNS records cover the whole domain), so any address
        // on the integration's domain is exactly as verified as the integration's own address.
        // Anything off-domain is discarded: honoring it would let a workflow send as a domain the
        // team never proved ownership of.
        const integrationDomain: string = (
            integration.config.domain ??
            integration.config.email.split('@')[1] ??
            ''
        ).toLowerCase()
        const overrideDomain = overrideEmail.split('@')[1].toLowerCase()
        if (!integrationDomain || overrideDomain !== integrationDomain) {
            addLog(
                'warn',
                `Ignoring the custom sender address "${overrideEmail}": it is not on the verified domain "${integrationDomain}" of the selected sender. Sending from ${integration.config.email} instead. Use an address on that domain, or select a different sender in the workflow's email step.`
            )
            return integration.config.email
        }

        return overrideEmail
    }

    private async loadAttachments(
        jobId: string,
        teamId: number,
        expected: CyclotronInvocationQueueParametersEmailType['attachments']
    ): Promise<DecodedAttachment[]> {
        if (!expected?.length) {
            return []
        }
        if (!this.emailAttachmentsService) {
            throw new EmailAttachmentsMissingError()
        }
        try {
            return await this.emailAttachmentsService.load(jobId, teamId, expected)
        } catch (error) {
            if (error instanceof EmailAttachmentsMissingError) {
                throw error
            }
            throw new EmailAttachmentsUnavailableError(error)
        }
    }

    // Send email to local maildev instance for testing (DEBUG=1 only)
    private async sendEmailWithMaildev(
        result: CyclotronJobInvocationResult<CyclotronJobInvocationHogFunction>,
        params: CyclotronInvocationQueueParametersEmailType,
        from: { email: string; name: string },
        trackingEnabled: boolean,
        isTest = false,
        files: DecodedAttachment[] = []
    ): Promise<void> {
        // This can timeout but there is no native timeout so we do our own one
        const mailOptions: SendMailOptions = {
            from: toNodemailerAddress(from.name, from.email),
            to: toNodemailerAddress(params.to.name, params.to.email),
            subject: sanitizeEmailSubject(params.subject),
            ...(params.text ? { text: params.text } : {}),
            headers: { [AUTO_SUBMITTED_HEADER.Name!]: AUTO_SUBMITTED_HEADER.Value! },
            ...(params.html
                ? {
                      html: trackingEnabled
                          ? addTrackingToEmail(params.html, result.invocation, this.trackingCodeSigner, isTest)
                          : params.html,
                  }
                : {}),
        }

        const ccAddresses = parseAddressList(params.cc)
        const bccAddresses = parseAddressList(params.bcc)
        const replyToAddresses = parseAddressList(params.replyTo)

        if (ccAddresses) {
            mailOptions.cc = ccAddresses
        }
        if (bccAddresses) {
            mailOptions.bcc = bccAddresses
        }
        if (replyToAddresses) {
            mailOptions.replyTo = replyToAddresses
        }
        if (files.length) {
            mailOptions.attachments = files.map((file) => ({
                filename: file.filename,
                content: file.content,
                contentType: file.contentType,
                cid: file.contentId,
                contentDisposition: file.contentId ? 'inline' : 'attachment',
            }))
        }

        const response = await mailDevTransport!.sendMail(mailOptions)

        if (!response.accepted) {
            throw new Error(`Failed to send email to maildev: ${JSON.stringify(response)}`)
        }

        result.logs.push(logEntry('debug', `Email sent to your local maildev server: ${mailDevWebUrl}`))
    }

    private async sendEmailWithSES(
        result: CyclotronJobInvocationResult<CyclotronJobInvocationHogFunction>,
        params: CyclotronInvocationQueueParametersEmailType,
        from: { email: string; name: string },
        trackingEnabled: boolean,
        isTest = false,
        queueWait?: EmailQueueWait,
        files: DecodedAttachment[] = []
    ): Promise<void> {
        if (!this.sesV2Client) {
            throw new Error('SES is not configured - set SES_REGION and AWS credentials')
        }
        const distinctId = resolveEmailTrackingDistinctId(result.invocation)
        // Full signed code (with distinct_id + isTest) rides in the header; the short unsigned
        // carrier (no distinct_id/isTest) goes in the SES EmailTag, guaranteed under the 256-char
        // tag-value limit. The webhook reads the header first and only falls back to the tag.
        const workflowVersion = resolveEmailSendingVersion(result.invocation)
        const trackingCode = this.trackingCodeSigner.generate(
            { ...result.invocation, distinctId, workflowVersion },
            isTest
        )
        const shortTrackingCode = this.trackingCodeSigner.generateShort(result.invocation)

        const htmlBody = params.html
            ? {
                  Html: {
                      Data: maybeAddPreheaderToEmail(
                          trackingEnabled
                              ? addTrackingToEmail(
                                    params.html,
                                    result.invocation,
                                    this.trackingCodeSigner,
                                    isTest,
                                    'ses'
                                )
                              : params.html,
                          params.preheader
                      ),
                      Charset: 'UTF-8',
                  },
              }
            : {}

        const sendEmailParams: SendEmailCommandInput = {
            FromEmailAddress: formatMailbox(from.name, from.email),
            Destination: {
                ToAddresses: [formatMailbox(params.to.name, params.to.email)],
            },
            Content: {
                Simple: {
                    Subject: {
                        Data: sanitizeEmailSubject(params.subject),
                        Charset: 'UTF-8',
                    },
                    Body: {
                        ...(params.text ? { Text: { Data: params.text, Charset: 'UTF-8' } } : {}),
                        ...htmlBody,
                    },
                },
            },
            ConfigurationSetName: this.resolveConfigurationSetName(trackingEnabled, result.invocation),
            // Short unsigned tag kept as a backwards-compat carrier for in-flight messages and
            // environments where the configuration set isn't yet emitting original headers.
            EmailTags: [
                { Name: 'ph_id', Value: shortTrackingCode },
                ...(result.invocation.hogFunction?.metadata?.sensitive === true
                    ? [{ Name: SES_SENSITIVE_TAG, Value: 'true' }]
                    : []),
                ...(!isTest && isTransactionalEmailSend(result.invocation)
                    ? [{ Name: SES_EMAIL_STATUS_TAG, Value: 'true' }]
                    : []),
            ],
            FeedbackForwardingEmailAddress: from.email,
        }

        // Attributes the send to the team's SES tenant so AWS tracks reputation per team and
        // its reputation policy can pause one tenant instead of the shared account. `team-<id>`
        // is the provisioning convention (products/workflows/backend/providers/ses.py and
        // posthog/management/commands/migrate_ses_tenants.py). Deliberately NOT gated on
        // isTest: test-panel sends are real over-the-wire SES sends, so leaving them
        // unattributed would (a) push their bounces onto the shared account's reputation and
        // (b) let a paused tenant keep sending via "Run test". The isTest skips elsewhere in
        // this class only shield our internal metrics, a separate concern from SES-side
        // attribution; test volume is far below the representative volume AWS needs for a
        // reputation finding.
        sendEmailParams.TenantName = `team-${result.invocation.teamId}`

        // Authoritative tracking-code carrier: a custom MIME header. Header values aren't
        // 256-char-bounded the way SES tag values are, so they safely carry the signed code
        // (with distinct_id). The configuration set's event destination needs
        // `IncludeOriginalHeaders: true` for the webhook to surface this header.
        const trackingHeader: MessageHeader = { Name: TRACKING_CODE_HEADER_NAME, Value: trackingCode }

        const isTransactionalEmail = result.invocation.hogFunction?.metadata?.message_category_type === 'transactional'
        if (sendEmailParams.Content?.Simple) {
            const unsubscribeHeaders = !isTransactionalEmail
                ? this.generateUnsubscribeHeaders(
                      { team_id: result.invocation.teamId, identifier: unquoteMailbox(params.to.email) },
                      preferencesTokenSourceForInvocation(result.invocation)
                  )
                : []
            sendEmailParams.Content.Simple.Headers = [...unsubscribeHeaders, trackingHeader, AUTO_SUBMITTED_HEADER]
            if (files.length) {
                sendEmailParams.Content.Simple.Attachments = files.map((file) => ({
                    RawContent: file.content,
                    FileName: file.filename,
                    ContentType: file.contentType,
                    ContentId: file.contentId,
                    ContentDisposition: file.contentId ? 'INLINE' : 'ATTACHMENT',
                    ContentTransferEncoding: 'BASE64',
                }))
            }
        }

        const replyToAddresses = parseAddressList(params.replyTo)
        const ccAddresses = parseAddressList(params.cc)
        const bccAddresses = parseAddressList(params.bcc)

        if (replyToAddresses) {
            sendEmailParams.ReplyToAddresses = replyToAddresses
        }
        if (ccAddresses) {
            sendEmailParams.Destination!.CcAddresses = ccAddresses
        }
        if (bccAddresses) {
            sendEmailParams.Destination!.BccAddresses = bccAddresses
        }

        if (queueWait) {
            emailQueueToSendMs
                .labels({
                    source: queueWait.source,
                    priority_class: getEmailQueuePriorityClass(result.invocation.hogFunction.metadata),
                    team_capped: queueWait.teamCapped ? 'true' : 'false',
                })
                .observe(Math.max(0, Date.now() - queueWait.startedAt.toMillis()))
        }

        const sendStartedAt = performance.now()
        try {
            const response = await this.sesV2Client.send(new SendEmailCommand(sendEmailParams))
            if (!response.MessageId) {
                throw new Error('No messageId returned from SES')
            }
            sesSendDurationMs.labels({ result: 'success' }).observe(performance.now() - sendStartedAt)
            sesSendResultsTotal.labels({ result: 'success' }).inc()
        } catch (error: unknown) {
            sesSendDurationMs.labels({ result: 'error' }).observe(performance.now() - sendStartedAt)
            sesSendResultsTotal.labels({ result: classifySesSendError(error) }).inc()
            if (isSesThrottleError(error)) {
                sesThrottleResponsesTotal.inc({ error_code: error.name })
                throw new SESThrottleError(error.name, pickThrottleRetryDelayMs(), error.message)
            }
            const message = error instanceof Error ? error.message : String(error)
            throw new Error(`Failed to send email via SES: ${message}`)
        }
    }

    private generateUnsubscribeHeaders(
        recipient: Pick<RecipientManagerRecipient, 'team_id' | 'identifier'>,
        source: PreferencesTokenSource | undefined
    ): MessageHeader[] {
        return [
            {
                Name: 'List-Unsubscribe',
                Value: `<${this.recipientTokensService.generateOneClickUnsubscribeUrl(recipient, source)}>`,
            },
            {
                Name: 'List-Unsubscribe-Post',
                Value: 'List-Unsubscribe=One-Click',
            },
        ]
    }
}