:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(cdp): refuse email attachments until they leave the cyclotron database (#130952)
Meikel Ratz33a95b49e8bd
diff --git a/nodejs/src/cdp/cdp-api.serial.test.ts b/nodejs/src/cdp/cdp-api.serial.test.tsindex f4dd9ccc48d4..36da26304897 100644--- a/nodejs/src/cdp/cdp-api.serial.test.ts+++ b/nodejs/src/cdp/cdp-api.serial.test.ts@@ -118,6 +118,7 @@ describe('CDP API', () => {SITE_URL: 'http://localhost:8000',})hub.CDP_GOOGLE_ADWORDS_DEVELOPER_TOKEN = 'ADWORDS_TOKEN'+ hub.CDP_EMAIL_ATTACHMENTS_ENABLED = trueteam = await getFirstTeam(hub.postgres)cdpDeps = createCdpConsumerDeps(hub)@@ -2681,6 +2682,42 @@ describe('CDP API', () => {}})+ it('refuses a new code send with files while attachments are off, and still replays one sent before', async () => {+ const sentBefore = {+ ...body,+ idempotency_key: `key-${new UUIDT().toString()}`,+ send: { ...codeSend, attachments: [attachment()] },+ }+ const accepted = await sendAccepted(hogFlow.id, sentBefore)+ hub.CDP_EMAIL_ATTACHMENTS_ENABLED = false+ const filename = `report-${new UUIDT().toString()}.pdf`+ const refusedKey = `key-${new UUIDT().toString()}`++ try {+ const replay = await sendAccepted(hogFlow.id, sentBefore)+ const refused = await sendAccepted(hogFlow.id, {+ ...body,+ idempotency_key: refusedKey,+ send: { ...codeSend, attachments: [attachment({ filename })] },+ })+ const withoutFiles = await sendAccepted(hogFlow.id, {+ ...body,+ idempotency_key: refusedKey,+ send: codeSend,+ })++ expect(replay.body).toMatchObject({ invocation_id: accepted.body.invocation_id, replayed: true })+ expect(refused.status).toEqual(400)+ expect(parseJSON(refused.body.error)).toEqual([+ expect.objectContaining({ code: 'unrecognized_keys', keys: ['attachments'], path: ['send'] }),+ ])+ expect(await storedAttachments(filename)).toEqual([])+ expect(withoutFiles.status).toEqual(202)+ } finally {+ hub.CDP_EMAIL_ATTACHMENTS_ENABLED = true+ }+ })+it('keeps the stored files when the queue insert fails, as the job may be queued already', async () => {mockQueueInvocations.mockRejectedValueOnce(new Error('queue down'))const filename = `report-${new UUIDT().toString()}.pdf`diff --git a/nodejs/src/cdp/cdp-api.ts b/nodejs/src/cdp/cdp-api.tsindex d31569ef5ba9..31dcaa72ef37 100644--- a/nodejs/src/cdp/cdp-api.ts+++ b/nodejs/src/cdp/cdp-api.ts@@ -64,6 +64,7 @@ import {TransactionalEmailRequestSchema,TransactionalEmailStageTimer,decodeAttachments,+ fieldNotAcceptedError,isRecipientFormatIssue,queuedSend,transactionalEmailRecipients,@@ -1340,6 +1341,14 @@ export class CdpApi {}}try {+ // After the idempotency check, so a retry of a send accepted with files still gets its answer.+ if (send?.attachments !== undefined && !this.config.CDP_EMAIL_ATTACHMENTS_ENABLED) {+ if (idempotencyKey) {+ stages.start('idempotency')+ await idempotencyKeys.release(team.id, idempotencyKey, invocation.id)+ }+ return reply('invalid_request', 400, { error: fieldNotAcceptedError('attachments') })+ }stages.start('checks')const refusal = await this.refuseTransactionalEmail(team.id,diff --git a/nodejs/src/cdp/config.ts b/nodejs/src/cdp/config.tsindex 4bba4d832a1b..fc379e0aaea7 100644--- a/nodejs/src/cdp/config.ts+++ b/nodejs/src/cdp/config.ts@@ -165,6 +165,9 @@ export type CdpConfig = ClickhouseConfig & {CDP_DLQ_REPLAY_TOPIC: stringCDP_EMAIL_TRACKING_URL: string+ // Attachment bytes are stored in the Cyclotron database, which the job queue also uses. While this is off,+ // the API refuses a send with `attachments` before it stores any file.+ CDP_EMAIL_ATTACHMENTS_ENABLED: boolean// Cyclotron (CDP job queue)CYCLOTRON_SHARD_DEPTH_LIMIT: number@@ -417,6 +420,7 @@ export function getDefaultCdpConfig(): CdpConfig {CDP_DLQ_REPLAY_TOPIC: KAFKA_CDP_EVENTS_DLQ,CDP_EMAIL_TRACKING_URL: 'http://localhost:8010',+ CDP_EMAIL_ATTACHMENTS_ENABLED: false,// CyclotronCYCLOTRON_SHARD_DEPTH_LIMIT: 1000000,diff --git a/nodejs/src/cdp/services/hogflows/transactional-email.ts b/nodejs/src/cdp/services/hogflows/transactional-email.tsindex 2d2292ea514c..1bb97f4ba60f 100644--- a/nodejs/src/cdp/services/hogflows/transactional-email.ts+++ b/nodejs/src/cdp/services/hogflows/transactional-email.ts@@ -227,6 +227,13 @@ export type AcceptedSend = z.infer<typeof AcceptedSendSchema>export type QueuedSend = Omit<AcceptedSend, 'attachments'> & { attachments?: readonly EmailAttachmentMetadata[] }+// Django reads an `unrecognized_keys` issue on `send` as a field this pipeline doesn't take. It refuses the+// send with that field named, so the caller sees which field to leave out.+export const fieldNotAcceptedError = (field: string): string =>+ JSON.stringify([+ { code: 'unrecognized_keys', keys: [field], path: ['send'], message: `Unrecognized key: "${field}"` },+ ])+export type DecodedAttachment = EmailAttachmentMetadata & { content: Buffer }export const decodeAttachments = (send: AcceptedSend | undefined): DecodedAttachment[] =>