Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

11.8 KB
Raw Blame
import { createHash } from 'crypto'
import { Counter, Histogram } from 'prom-client'
import { z } from 'zod'

import {
    CyclotronInvocationQueueParametersEmailType,
    EmailAttachmentMetadata,
    MAX_EMAIL_ATTACHMENTS,
} from '~/cdp/schema/cyclotron'

export const MAX_COPY_RECIPIENTS = 10

export const RECIPIENT_ADDRESS_PATTERN =
    /^[a-zA-Z0-9!#$%&'*+/=?^_`{|}~-]+(?:\.[a-zA-Z0-9!#$%&'*+/=?^_`{|}~-]+)*@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)+$/

const CopyRecipientsSchema = z
    .array(z.string().email({ pattern: RECIPIENT_ADDRESS_PATTERN }))
    .max(MAX_COPY_RECIPIENTS)
    .readonly()
    .optional()

export const MAX_EMAIL_ATTACHMENTS_BYTES = 512 * 1024

const BLOCKED_ATTACHMENT_EXTENSIONS: ReadonlySet<string> = new Set([
    'ade',
    'adp',
    'apk',
    'app',
    'appx',
    'appxbundle',
    'asp',
    'bas',
    'bat',
    'cab',
    'cer',
    'chm',
    'cmd',
    'com',
    'cpl',
    'crt',
    'csh',
    'der',
    'diagcab',
    'diagcfg',
    'diagpkg',
    'dll',
    'dmg',
    'ex',
    'ex_',
    'exe',
    'fxp',
    'gadget',
    'hlp',
    'hta',
    'img',
    'inf',
    'ins',
    'iso',
    'isp',
    'its',
    'jar',
    'jnlp',
    'js',
    'jse',
    'ksh',
    'lib',
    'lnk',
    'mad',
    'maf',
    'mag',
    'mam',
    'maq',
    'mar',
    'mas',
    'mat',
    'mau',
    'mav',
    'maw',
    'mda',
    'mdb',
    'mde',
    'mdt',
    'mdw',
    'mdz',
    'mjs',
    'msc',
    'msh',
    'msh1',
    'msh1xml',
    'msh2',
    'msh2xml',
    'mshxml',
    'msi',
    'msix',
    'msixbundle',
    'msp',
    'mst',
    'nsh',
    'ops',
    'pcd',
    'pif',
    'plg',
    'prf',
    'prg',
    'ps1',
    'ps1xml',
    'ps2',
    'ps2xml',
    'psc1',
    'psc2',
    'reg',
    'scf',
    'scr',
    'sct',
    'shb',
    'shs',
    'sys',
    'tmp',
    'url',
    'vb',
    'vbe',
    'vbs',
    'vhd',
    'vps',
    'vsmacros',
    'vss',
    'vst',
    'vsw',
    'vxd',
    'ws',
    'wsc',
    'wsf',
    'wsh',
    'xll',
    'xnk',
])

const BASE64_RE = /^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/
const MAX_ATTACHMENT_BASE64_LENGTH = Math.ceil(MAX_EMAIL_ATTACHMENTS_BYTES / 3) * 4
const MIME_TYPE_RE = /^[a-z0-9][a-z0-9!#$&^_.+-]{0,126}\/[a-z0-9][a-z0-9!#$&^_.+-]{0,126}$/i
const CONTROL_CHARACTER_RE = /[\x00-\x1F\x7F]/
const SES_ATTACHMENT_HEADER_MAX_LENGTH = 78

const AttachmentSchema = z
    .object({
        filename: z
            .string()
            .min(1)
            .max(255)
            .refine((name) => !/[\\/]/.test(name) && !CONTROL_CHARACTER_RE.test(name), {
                message: 'filename must not contain a path separator or a line break',
            })
            .refine(
                (name) => {
                    const dot = name.lastIndexOf('.')
                    return dot < 0 || !BLOCKED_ATTACHMENT_EXTENSIONS.has(name.slice(dot + 1).toLowerCase())
                },
                { message: 'mailbox providers block this file type' }
            ),
        content: z
            .string()
            .min(1, { message: 'content must not be empty' })
            .max(MAX_ATTACHMENT_BASE64_LENGTH, { message: 'attachments must be 512 KB or less in total' })
            .refine((content) => content.length <= MAX_ATTACHMENT_BASE64_LENGTH && BASE64_RE.test(content), {
                message: 'content must be base64',
            }),
        contentType: z
            .string()
            .max(SES_ATTACHMENT_HEADER_MAX_LENGTH)
            .regex(MIME_TYPE_RE, { message: 'contentType must be a MIME type' })
            .optional(),
        contentId: z
            .string()
            .min(1)
            .max(SES_ATTACHMENT_HEADER_MAX_LENGTH)
            .refine((id) => !/[<>]/.test(id) && !CONTROL_CHARACTER_RE.test(id), {
                message: 'contentId must not contain < or > or a line break',
            })
            .optional(),
    })
    .strict()
    .readonly()

const AttachmentsSchema = z
    .array(AttachmentSchema)
    .max(MAX_EMAIL_ATTACHMENTS)
    .refine(
        (files) =>
            files.reduce((total, file) => total + Buffer.byteLength(file.content, 'base64'), 0) <=
            MAX_EMAIL_ATTACHMENTS_BYTES,
        { message: 'attachments must be 512 KB or less in total' }
    )
    .readonly()
    .optional()

export const AcceptedSendSchema = z
    .object({
        content: z
            .object({
                subject: z.string().min(1),
                html: z.string().optional(),
                text: z.string().optional(),
            })
            .strict()
            .readonly()
            .refine((content) => !!content.html || !!content.text, { message: 'content needs html or text' })
            .optional(),
        from: z
            .object({
                integrationId: z.number().int(),
                email: z.string().optional(),
                name: z.string().optional(),
            })
            .strict()
            .readonly()
            .optional(),
        replyTo: z.array(z.string()).readonly().optional(),
        cc: CopyRecipientsSchema,
        bcc: CopyRecipientsSchema,
        attachments: AttachmentsSchema,
        tracking: z.object({ captureEngagement: z.boolean() }).strict().readonly(),
    })
    .strict()
    .readonly()

export type AcceptedSend = z.infer<typeof AcceptedSendSchema>

export type QueuedSend = Omit<AcceptedSend, 'attachments'> & { attachments?: readonly EmailAttachmentMetadata[] }

// Django reads an `unrecognized_keys` issue on `send` as a field this pipeline doesn't take. It refuses the
// send with that field named, so the caller sees which field to leave out.
export const fieldNotAcceptedError = (field: string): string =>
    JSON.stringify([
        { code: 'unrecognized_keys', keys: [field], path: ['send'], message: `Unrecognized key: "${field}"` },
    ])

export type DecodedAttachment = EmailAttachmentMetadata & { content: Buffer }

export const decodeAttachments = (send: AcceptedSend | undefined): DecodedAttachment[] =>
    (send?.attachments ?? []).map(({ filename, content, contentType, contentId }) => {
        const bytes = Buffer.from(content, 'base64')
        return {
            filename,
            ...(contentType !== undefined && { contentType }),
            ...(contentId !== undefined && { contentId }),
            sizeBytes: bytes.length,
            sha256: createHash('sha256').update(bytes).digest('hex'),
            content: bytes,
        }
    })

export const attachmentMetadata = (files: DecodedAttachment[]): EmailAttachmentMetadata[] =>
    files.map(({ content: _content, ...metadata }) => metadata)

export const queuedSend = (send: AcceptedSend, files: DecodedAttachment[]): QueuedSend => {
    const { attachments, ...rest } = send
    return attachments === undefined ? rest : { ...rest, attachments: attachmentMetadata(files) }
}

export const acceptedSendEmailFields = (
    send: QueuedSend | undefined
): Partial<CyclotronInvocationQueueParametersEmailType> => ({
    ...(send?.content && {
        subject: send.content.subject,
        html: send.content.html ?? '',
        text: send.content.text ?? '',
        preheader: '',
    }),
    ...(send?.from && { from: { ...send.from } }),
    ...(send?.replyTo && { replyTo: send.replyTo.join(', ') }),
    ...(send?.cc && { cc: send.cc.join(', ') }),
    ...(send?.bcc && { bcc: send.bcc.join(', ') }),
    ...(send?.attachments && { attachments: send.attachments.map((file) => ({ ...file })) }),
})

// What Django sends to run a transactional email's workflow once.
export const TransactionalEmailRequestSchema = z.object({
    to: z.string().email({ pattern: RECIPIENT_ADDRESS_PATTERN }),
    // The person the send is tracked against. Django falls back to the recipient's address.
    distinct_id: z.string().min(1),
    // The caller's `message_data`, available to the template as `variables`.
    variables: z.record(z.string(), z.any()).default({}),
    // The caller's `Idempotency-Key`. A repeat with the same key and request returns the first send.
    idempotency_key: z.string().min(1).max(255).optional(),
})

export const TEMPLATE_SEND_TOKEN_CLAIMS = { kind: undefined }
export const CODE_SEND_TOKEN_CLAIMS = { kind: 'code_send' }

export const AcceptedSendRequestSchema = TransactionalEmailRequestSchema.extend({ send: AcceptedSendSchema })

export type TransactionalEmailRequest = z.infer<typeof TransactionalEmailRequestSchema> & { send?: AcceptedSend }

export const transactionalEmailRecipients = ({ to, send }: TransactionalEmailRequest): string[] => [
    to,
    ...(send?.cc ?? []),
    ...(send?.bcc ?? []),
]

export const isRecipientFormatIssue = (issue: z.core.$ZodIssue): boolean =>
    issue.code === 'invalid_format' &&
    (issue.path[0] === 'to' || (issue.path[0] === 'send' && (issue.path[1] === 'cc' || issue.path[1] === 'bcc')))

export type TransactionalEmailOutcome =
    | 'queued'
    | 'replayed'
    | 'idempotency_key_in_flight'
    | 'idempotency_key_reused'
    | 'idempotency_unavailable'
    | 'not_configured'
    | 'unauthorized'
    | 'invalid_request'
    | 'not_found'
    | 'not_sendable'
    | 'quota_limited'
    | 'sending_suspended'
    | 'sending_paused'
    | 'recipient_suppressed'
    | 'recipient_unreachable'
    | 'error'

export const transactionalEmailRequestsTotal = new Counter({
    name: 'cdp_transactional_email_requests_total',
    help: 'Requests from Django to run a transactional email, by outcome. Every outcome except `queued` means this request queued no email.',
    labelNames: ['outcome'],
})

export const TRANSACTIONAL_EMAIL_STAGES = [
    'auth',
    'validate',
    'load',
    'build',
    'idempotency',
    'checks',
    'files',
    'queue_insert',
    'metrics_flush',
] as const
export type TransactionalEmailStage = (typeof TRANSACTIONAL_EMAIL_STAGES)[number]

export const transactionalEmailStageDurationMs = new Histogram({
    name: 'cdp_transactional_email_stage_duration_ms',
    help: 'Milliseconds one transactional email request spends in each stage: auth (the JWT), validate (the body), load (the team and the workflow), build (the run and its idempotency fingerprint), idempotency (every Idempotency-Key call to Valkey), checks (suspension, pause, suppression, quota and MX), files (the attachment rows written to the Cyclotron database), queue_insert (the email queue insert), metrics_flush (the hand-off of the result and status rows to the Kafka producer, without the wait for the broker), and total. A request records only the stages it reached, so a refusal shows in the stage that refused it.',
    labelNames: ['stage'],
    buckets: [1, 2.5, 5, 10, 25, 50, 100, 250, 500, 1_000, 2_500, 5_000, 10_000],
})

// The stages of a request run back to back, so starting a stage ends the one before it. finish() records each stage
// once, with its summed time, so a stage entered twice (idempotency) still counts as one observation per request.
export class TransactionalEmailStageTimer {
    private readonly startedAt = performance.now()
    private current: { stage: TransactionalEmailStage; startedAt: number } | null = null
    private readonly durations = new Map<TransactionalEmailStage, number>()

    start(stage: TransactionalEmailStage): void {
        const now = performance.now()
        this.endCurrent(now)
        this.current = { stage, startedAt: now }
    }

    finish(): void {
        const now = performance.now()
        this.endCurrent(now)
        for (const [stage, durationMs] of this.durations) {
            transactionalEmailStageDurationMs.labels({ stage }).observe(durationMs)
        }
        transactionalEmailStageDurationMs.labels({ stage: 'total' }).observe(now - this.startedAt)
    }

    private endCurrent(now: number): void {
        if (this.current) {
            const { stage, startedAt } = this.current
            this.durations.set(stage, (this.durations.get(stage) ?? 0) + now - startedAt)
            this.current = null
        }
    }
}