Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(workflows): report blocked and failed workflow sends (#127135)

Michael Reichenbach5fd7b11654da
diff --git a/nodejs/src/cdp/cdp-services.ts b/nodejs/src/cdp/cdp-services.ts
index c3bda80b7084..79187db4a904 100644
--- a/nodejs/src/cdp/cdp-services.ts
+++ b/nodejs/src/cdp/cdp-services.ts
@@ -47,6 +47,7 @@ import { RecipientTokensService } from './services/messaging/recipient-tokens.se
import { HogFunctionMonitoringService } from './services/monitoring/hog-function-monitoring.service'
import { HogInvocationResultsService } from './services/monitoring/hog-invocation-results.service'
import { HogWatcherService } from './services/monitoring/hog-watcher.service'
+import { WorkflowsActivationReporter } from './services/monitoring/workflows-activation-reporter'
import { NativeDestinationExecutorService } from './services/native-destination-executor.service'
import { RateLimiterService } from './services/rate-limiter/rate-limiter.service'
import { SegmentDestinationExecutorService } from './services/segment-destination-executor.service'
@@ -106,6 +107,7 @@ export interface CdpCoreServices {
cohortMembershipRepository: CohortMembershipRepository
hogFlowExecutor: HogFlowExecutorService
hogFunctionMonitoringService: HogFunctionMonitoringService
+ workflowsActivationReporter: WorkflowsActivationReporter
cdpUsageReporter: CdpUsageReporterService
capturedEventsService: CapturedEventsService
/** Per-invocation lifecycle row producer for the new runs/invocations UI + rerun path. */
@@ -453,6 +455,7 @@ export function createCdpCoreServices(
})
const recipientsManager = new RecipientsManagerService(deps.postgres)
const personsManager = new PersonsManagerService(deps.teamManager, deps.personRepository, config.SITE_URL)
+ const workflowsActivationReporter = new WorkflowsActivationReporter(deps.teamManager)
// Per-workflow send pacing rides the SES Valkey pool, which is only populated on pods that
// execute email actions — exactly where the limit is enforced. Null elsewhere disables it.
const workflowEmailRateLimiter = deps.emailValidationValkey
@@ -483,6 +486,7 @@ export function createCdpCoreServices(
emailSuppressionService,
recipientsManager,
emailStatusService,
+ workflowsActivationReporter,
messageAssetsService,
workflowEmailRateLimiter,
teamEmailRateLimiter
@@ -624,6 +628,7 @@ export function createCdpCoreServices(
cohortMembershipRepository,
hogFlowExecutor,
hogFunctionMonitoringService,
+ workflowsActivationReporter,
cdpUsageReporter,
capturedEventsService,
hogInvocationResultsService,
diff --git a/nodejs/src/cdp/consumers/cdp-base.consumer.ts b/nodejs/src/cdp/consumers/cdp-base.consumer.ts
index c27500666b02..62ed7859cec3 100644
--- a/nodejs/src/cdp/consumers/cdp-base.consumer.ts
+++ b/nodejs/src/cdp/consumers/cdp-base.consumer.ts
@@ -31,6 +31,7 @@ import { RecipientPreferencesService } from '../services/messaging/recipient-pre
import { HogFunctionMonitoringService } from '../services/monitoring/hog-function-monitoring.service'
import { HogMaskerService } from '../services/monitoring/hog-masker.service'
import { HogWatcherService } from '../services/monitoring/hog-watcher.service'
+import { WorkflowsActivationReporter } from '../services/monitoring/workflows-activation-reporter'
import { NativeDestinationExecutorService } from '../services/native-destination-executor.service'
import { SegmentDestinationExecutorService } from '../services/segment-destination-executor.service'
import { CdpUsageReporterService } from '../services/usage/cdp-usage-reporter.service'
@@ -73,6 +74,7 @@ export abstract class CdpConsumerBase<TConfig extends CdpConsumerBaseConfig = Cd
emailService: EmailService
hogFunctionMonitoringService: HogFunctionMonitoringService
+ workflowsActivationReporter: WorkflowsActivationReporter
cdpUsageReporter: CdpUsageReporterService
invocationResultsService: InvocationResultsService
nativeDestinationExecutorService: NativeDestinationExecutorService
@@ -105,6 +107,7 @@ export abstract class CdpConsumerBase<TConfig extends CdpConsumerBaseConfig = Cd
this.hogFlowExecutor = services.hogFlowExecutor
this.emailService = services.emailService
this.hogFunctionMonitoringService = services.hogFunctionMonitoringService
+ this.workflowsActivationReporter = services.workflowsActivationReporter
this.cdpUsageReporter = services.cdpUsageReporter
this.invocationResultsService = services.invocationResultsService
this.nativeDestinationExecutorService = services.nativeDestinationExecutorService
diff --git a/nodejs/src/cdp/consumers/cdp-cohort-membership-trigger.consumer.ts b/nodejs/src/cdp/consumers/cdp-cohort-membership-trigger.consumer.ts
index a52e697fe750..5538e8df5b1a 100644
--- a/nodejs/src/cdp/consumers/cdp-cohort-membership-trigger.consumer.ts
+++ b/nodejs/src/cdp/consumers/cdp-cohort-membership-trigger.consumer.ts
@@ -102,6 +102,7 @@ export class CdpCohortMembershipTriggerConsumer extends CdpConsumerBase {
hogWatcherMirror: this.hogWatcherMirror,
hogMasker: this.hogMasker,
hogFunctionMonitoringService: this.hogFunctionMonitoringService,
+ workflowsActivationReporter: this.workflowsActivationReporter,
quotaLimiting: deps.quotaLimiting,
redis: this.redis,
valkeyShadow: this.valkeyShadow,
diff --git a/nodejs/src/cdp/consumers/cdp-data-warehouse-events.consumer.ts b/nodejs/src/cdp/consumers/cdp-data-warehouse-events.consumer.ts
index 2fe93daf3416..66016324e615 100644
--- a/nodejs/src/cdp/consumers/cdp-data-warehouse-events.consumer.ts
+++ b/nodejs/src/cdp/consumers/cdp-data-warehouse-events.consumer.ts
@@ -73,6 +73,7 @@ export class CdpDatawarehouseEventsConsumer extends CdpConsumerBase {
hogWatcherMirror: this.hogWatcherMirror,
hogMasker: this.hogMasker,
hogFunctionMonitoringService: this.hogFunctionMonitoringService,
+ workflowsActivationReporter: this.workflowsActivationReporter,
quotaLimiting: deps.quotaLimiting,
redis: this.redis,
valkeyShadow: this.valkeyShadow,
diff --git a/nodejs/src/cdp/consumers/cdp-dlq-replay.consumer.ts b/nodejs/src/cdp/consumers/cdp-dlq-replay.consumer.ts
index 2f8a787c6a73..2b7bdcfc5b83 100644
--- a/nodejs/src/cdp/consumers/cdp-dlq-replay.consumer.ts
+++ b/nodejs/src/cdp/consumers/cdp-dlq-replay.consumer.ts
@@ -151,6 +151,7 @@ export class CdpDlqReplayConsumer extends CdpConsumerBase<PluginsServerConfig> {
hogWatcherMirror: this.hogWatcherMirror,
hogMasker: this.hogMasker,
hogFunctionMonitoringService: this.hogFunctionMonitoringService,
+ workflowsActivationReporter: this.workflowsActivationReporter,
quotaLimiting: deps.quotaLimiting,
redis: this.redis,
valkeyShadow: this.valkeyShadow,
diff --git a/nodejs/src/cdp/consumers/cdp-events.consumer.ts b/nodejs/src/cdp/consumers/cdp-events.consumer.ts
index 922b44dd7b49..a6cbbfaf04fc 100644
--- a/nodejs/src/cdp/consumers/cdp-events.consumer.ts
+++ b/nodejs/src/cdp/consumers/cdp-events.consumer.ts
@@ -71,6 +71,7 @@ export class CdpEventsConsumer<
hogWatcherMirror: this.hogWatcherMirror,
hogMasker: this.hogMasker,
hogFunctionMonitoringService: this.hogFunctionMonitoringService,
+ workflowsActivationReporter: this.workflowsActivationReporter,
quotaLimiting: deps.quotaLimiting,
redis: this.redis,
valkeyShadow: this.valkeyShadow,
diff --git a/nodejs/src/cdp/consumers/cdp-internal-event.consumer.ts b/nodejs/src/cdp/consumers/cdp-internal-event.consumer.ts
index 22c703c741a3..678b5c6f6f67 100644
--- a/nodejs/src/cdp/consumers/cdp-internal-event.consumer.ts
+++ b/nodejs/src/cdp/consumers/cdp-internal-event.consumer.ts
@@ -78,6 +78,7 @@ export class CdpInternalEventsConsumer extends CdpConsumerBase {
hogWatcherMirror: this.hogWatcherMirror,
hogMasker: this.hogMasker,
hogFunctionMonitoringService: this.hogFunctionMonitoringService,
+ workflowsActivationReporter: this.workflowsActivationReporter,
quotaLimiting: deps.quotaLimiting,
redis: this.redis,
valkeyShadow: this.valkeyShadow,
diff --git a/nodejs/src/cdp/services/hog-executor.service.test.ts b/nodejs/src/cdp/services/hog-executor.service.test.ts
index d110b6434e04..41bec369ca49 100644
--- a/nodejs/src/cdp/services/hog-executor.service.test.ts
+++ b/nodejs/src/cdp/services/hog-executor.service.test.ts
@@ -89,7 +89,8 @@ describe('Hog Executor', () => {
new EmailTrackingCodeSigner(hub.ENCRYPTION_SALT_KEYS, hub.CDP_EMAIL_TRACKING_URL),
new EmailSuppressionService(hub.postgres, emailSuppressionConfigFromEnv()),
new RecipientsManagerService(hub.postgres),
- null
+ null,
+ { report: jest.fn() }
)
const recipientTokensService = new RecipientTokensService(hub.ENCRYPTION_SALT_KEYS, hub.SITE_URL)
return new HogExecutorAsyncService(
diff --git a/nodejs/src/cdp/services/hog-flow-invocation-pipeline.service.test.ts b/nodejs/src/cdp/services/hog-flow-invocation-pipeline.service.test.ts
index 0e3a67ebe439..8220ecea846a 100644
--- a/nodejs/src/cdp/services/hog-flow-invocation-pipeline.service.test.ts
+++ b/nodejs/src/cdp/services/hog-flow-invocation-pipeline.service.test.ts
@@ -49,6 +49,7 @@ describe('HogFlowInvocationPipeline', () => {
let hogMasker: jest.Mocked<HogMaskerService>
let hogFunctionMonitoringService: jest.Mocked<HogFunctionMonitoringService>
let quotaLimiting: jest.Mocked<QuotaLimiting>
+ let workflowsActivationReporter: { report: jest.Mock }
let pipeline: HogFlowInvocationPipeline
let rateLimitGroupedMock: jest.Mock
@@ -83,6 +84,8 @@ describe('HogFlowInvocationPipeline', () => {
isTeamQuotaLimited: jest.fn().mockResolvedValue(false),
} as unknown as jest.Mocked<QuotaLimiting>
+ workflowsActivationReporter = { report: jest.fn() }
+
pipeline = new HogFlowInvocationPipeline(config, {
hogFlowManager,
hogFlowExecutor,
@@ -90,6 +93,7 @@ describe('HogFlowInvocationPipeline', () => {
hogWatcherMirror: hogWatcher,
hogMasker,
hogFunctionMonitoringService,
+ workflowsActivationReporter,
quotaLimiting,
redis: {} as any,
valkeyShadow: { writer: {} as any, reader: {} as any },
@@ -175,6 +179,11 @@ describe('HogFlowInvocationPipeline', () => {
const result = await pipeline.buildInvocations([makeGlobals()])
expect(result).toEqual([])
+ expect(workflowsActivationReporter.report).toHaveBeenCalledWith(inv.teamId, 'workflows send blocked', {
+ reason: 'quota_limited',
+ channel: 'email',
+ workflow_id: inv.functionId,
+ })
})
it('drops invocations for disabled hogflows', async () => {
diff --git a/nodejs/src/cdp/services/hog-flow-invocation-pipeline.service.ts b/nodejs/src/cdp/services/hog-flow-invocation-pipeline.service.ts
index 6d91fb01172d..bc016ff9f800 100644
--- a/nodejs/src/cdp/services/hog-flow-invocation-pipeline.service.ts
+++ b/nodejs/src/cdp/services/hog-flow-invocation-pipeline.service.ts
@@ -25,6 +25,7 @@ import { shouldBlockHogFlowDueToQuota } from './hogflows/hogflow-quota-limiting'
import { HogFunctionMonitoringService } from './monitoring/hog-function-monitoring.service'
import { HogMaskerService } from './monitoring/hog-masker.service'
import { HogWatcherService, HogWatcherState, sameWatcherStates } from './monitoring/hog-watcher.service'
+import { WorkflowsActivationReporter } from './monitoring/workflows-activation-reporter'
export interface HogFlowInvocationPipelineConfig {
CDP_RATE_LIMITER_BUCKET_SIZE: number
@@ -39,6 +40,7 @@ export interface HogFlowInvocationPipelineDeps {
hogWatcherMirror: HogWatcherService
hogMasker: HogMaskerService
hogFunctionMonitoringService: HogFunctionMonitoringService
+ workflowsActivationReporter: Pick<WorkflowsActivationReporter, 'report'>
quotaLimiting: QuotaLimiting
redis: RedisV2
valkeyShadow: CdpValkeyShadowPools
@@ -210,6 +212,7 @@ export class HogFlowInvocationPipeline {
const isQuotaLimited = await shouldBlockHogFlowDueToQuota(item, {
quotaLimiting: this.deps.quotaLimiting,
hogFunctionMonitoringService: this.deps.hogFunctionMonitoringService,
+ workflowsActivationReporter: this.deps.workflowsActivationReporter,
})
if (isQuotaLimited) {
diff --git a/nodejs/src/cdp/services/hogflows/actions/hog_function.test.ts b/nodejs/src/cdp/services/hogflows/actions/hog_function.test.ts
index bdb88d9c8caf..6ac510fb1bbd 100644
--- a/nodejs/src/cdp/services/hogflows/actions/hog_function.test.ts
+++ b/nodejs/src/cdp/services/hogflows/actions/hog_function.test.ts
@@ -93,7 +93,8 @@ describe('HogFunctionHandler', () => {
new EmailTrackingCodeSigner(hub.ENCRYPTION_SALT_KEYS, hub.CDP_EMAIL_TRACKING_URL),
new EmailSuppressionService(hub.postgres, emailSuppressionConfigFromEnv()),
new RecipientsManagerService(hub.postgres),
- null
+ null,
+ { report: jest.fn() }
)
mockHogFunctionExecutor = new HogExecutorAsyncService(
new HogExecutorService({ executionTimeoutMs: hub.CDP_WATCHER_HOG_COST_TIMING_UPPER_MS }, hogInputsService),
diff --git a/nodejs/src/cdp/services/hogflows/hogflow-executor.service.test.ts b/nodejs/src/cdp/services/hogflows/hogflow-executor.service.test.ts
index c632c352bf77..49c0688590be 100644
--- a/nodejs/src/cdp/services/hogflows/hogflow-executor.service.test.ts
+++ b/nodejs/src/cdp/services/hogflows/hogflow-executor.service.test.ts
@@ -100,7 +100,8 @@ describe('Hogflow Executor', () => {
new EmailTrackingCodeSigner(hub.ENCRYPTION_SALT_KEYS, hub.CDP_EMAIL_TRACKING_URL),
emailSuppressionService,
new RecipientsManagerService(hub.postgres),
- null
+ null,
+ { report: jest.fn() }
)
const recipientTokensService = new RecipientTokensService(hub.ENCRYPTION_SALT_KEYS, hub.SITE_URL)
const hogExecutor = new HogExecutorAsyncService(
diff --git a/nodejs/src/cdp/services/hogflows/hogflow-quota-limiting.test.ts b/nodejs/src/cdp/services/hogflows/hogflow-quota-limiting.test.ts
index ed38724827a8..ac02ae9d8849 100644
--- a/nodejs/src/cdp/services/hogflows/hogflow-quota-limiting.test.ts
+++ b/nodejs/src/cdp/services/hogflows/hogflow-quota-limiting.test.ts
@@ -216,6 +216,7 @@ describe('HogFlow Quota Limiting', () => {
describe('shouldBlockHogFlowDueToQuota', () => {
let mockHogFunctionMonitoringService: jest.Mocked<HogFunctionMonitoringService>
+ let mockWorkflowsActivationReporter: { report: jest.Mock }
const teamId = 123
const baseItem: CyclotronJobInvocationHogFlow = {
teamId,
@@ -238,6 +239,7 @@ describe('HogFlow Quota Limiting', () => {
mockHogFunctionMonitoringService = {
queueAppMetric: jest.fn(),
} as any
+ mockWorkflowsActivationReporter = { report: jest.fn() }
// Reset the counter spy
jest.spyOn(counterHogFlowQuotaLimited, 'labels').mockReturnValue({
inc: jest.fn(),
@@ -258,29 +260,58 @@ describe('HogFlow Quota Limiting', () => {
const result = await shouldBlockHogFlowDueToQuota(item, {
quotaLimiting: mockQuotaLimiting,
hogFunctionMonitoringService: mockHogFunctionMonitoringService,
+ workflowsActivationReporter: mockWorkflowsActivationReporter,
})
expect(result).toBe(false)
expect(mockHogFunctionMonitoringService.queueAppMetric).not.toHaveBeenCalled()
expect(counterHogFlowQuotaLimited.labels).not.toHaveBeenCalled()
+ expect(mockWorkflowsActivationReporter.report).not.toHaveBeenCalled()
})
- it('should block invocation and emit metrics when quota limited', async () => {
+ it.each([
+ [
+ 'an email send on the email quota',
+ 'workflow_emails',
+ ['function_email'],
+ [
+ [
+ teamId,
+ 'workflows send blocked',
+ { reason: 'quota_limited', channel: 'email', workflow_id: 'test-flow-id' },
+ ],
+ ],
+ ],
+ ['a webhook on the destination quota', 'workflow_destinations_dispatched', ['function'], []],
+ [
+ 'an email send next to a webhook on the destination quota',
+ 'workflow_destinations_dispatched',
+ ['function_email', 'function'],
+ [
+ [
+ teamId,
+ 'workflows send blocked',
+ { reason: 'quota_limited', channel: 'email', workflow_id: 'test-flow-id' },
+ ],
+ ],
+ ],
+ ] as const)('should block %s and emit metrics', async (_name, limitedResource, actionTypes, reports) => {
mockQuotaLimiting.isTeamQuotaLimited.mockImplementation((_teamId, resource) => {
- return Promise.resolve(resource === 'workflow_emails')
+ return Promise.resolve(resource === limitedResource)
})
const item: CyclotronJobInvocationHogFlow = {
...baseItem,
hogFlow: {
...baseItem.hogFlow,
- billable_action_types: ['function_email'],
+ billable_action_types: [...actionTypes],
},
}
const result = await shouldBlockHogFlowDueToQuota(item, {
quotaLimiting: mockQuotaLimiting,
hogFunctionMonitoringService: mockHogFunctionMonitoringService,
+ workflowsActivationReporter: mockWorkflowsActivationReporter,
})
expect(result).toBe(true)
@@ -296,6 +327,36 @@ describe('HogFlow Quota Limiting', () => {
},
'hog_flow'
)
+ expect(mockWorkflowsActivationReporter.report.mock.calls).toEqual(reports)
+ })
+
+ it('should report a blocked email send when the destination quota stops transactional emails next to a webhook', async () => {
+ mockQuotaLimiting.isTeamQuotaLimited.mockResolvedValue(true)
+
+ const item: CyclotronJobInvocationHogFlow = {
+ ...baseItem,
+ hogFlow: {
+ ...baseItem.hogFlow,
+ actions: [
+ { type: 'function_email', config: { message_category_type: 'transactional' } },
+ { type: 'function', config: {} },
+ ],
+ billable_action_types: ['function_email', 'function'],
+ } as unknown as HogFlow,
+ }
+
+ const result = await shouldBlockHogFlowDueToQuota(item, {
+ quotaLimiting: mockQuotaLimiting,
+ hogFunctionMonitoringService: mockHogFunctionMonitoringService,
+ workflowsActivationReporter: mockWorkflowsActivationReporter,
+ })
+
+ expect(result).toBe(true)
+ expect(mockWorkflowsActivationReporter.report).toHaveBeenCalledWith(teamId, 'workflows send blocked', {
+ reason: 'quota_limited',
+ channel: 'email',
+ workflow_id: 'test-flow-id',
+ })
})
it('should handle workflow with no billable actions', async () => {
@@ -312,6 +373,7 @@ describe('HogFlow Quota Limiting', () => {
const result = await shouldBlockHogFlowDueToQuota(item, {
quotaLimiting: mockQuotaLimiting,
hogFunctionMonitoringService: mockHogFunctionMonitoringService,
+ workflowsActivationReporter: mockWorkflowsActivationReporter,
})
expect(result).toBe(false)
diff --git a/nodejs/src/cdp/services/hogflows/hogflow-quota-limiting.ts b/nodejs/src/cdp/services/hogflows/hogflow-quota-limiting.ts
index 849173a89e80..f04e35eb0774 100644
--- a/nodejs/src/cdp/services/hogflows/hogflow-quota-limiting.ts
+++ b/nodejs/src/cdp/services/hogflows/hogflow-quota-limiting.ts
@@ -5,6 +5,7 @@ import { HogFlow, HogFlowAction } from '~/cdp/schema/hogflow'
import { QuotaLimiting } from '../../../common/services/quota-limiting.service'
import { CyclotronJobInvocationHogFlow } from '../../types'
import { HogFunctionMonitoringService } from '../monitoring/hog-function-monitoring.service'
+import { WorkflowsActivationReporter } from '../monitoring/workflows-activation-reporter'
export const counterHogFlowQuotaLimited = new Counter({
name: 'cdp_hog_flow_quota_limited',
@@ -73,6 +74,11 @@ export async function checkHogFlowQuotaLimits(
export interface HogFlowQuotaLimitingContext {
quotaLimiting: QuotaLimiting
hogFunctionMonitoringService: HogFunctionMonitoringService
+ workflowsActivationReporter: Pick<WorkflowsActivationReporter, 'report'>
+}
+
+function sendsEmail(hogFlow: HogFlow): boolean {
+ return Array.isArray(hogFlow.billable_action_types) && hogFlow.billable_action_types.includes('function_email')
}
/**
@@ -99,6 +105,13 @@ export async function shouldBlockHogFlowDueToQuota(
},
'hog_flow'
)
+ if (sendsEmail(item.hogFlow)) {
+ void context.workflowsActivationReporter.report(item.teamId, 'workflows send blocked', {
+ reason: 'quota_limited',
+ channel: 'email',
+ workflow_id: item.functionId,
+ })
+ }
return true
}
diff --git a/nodejs/src/cdp/services/messaging/email.service.test.ts b/nodejs/src/cdp/services/messaging/email.service.test.ts
index d42d6399c11a..836d3ca98b86 100644
--- a/nodejs/src/cdp/services/messaging/email.service.test.ts
+++ b/nodejs/src/cdp/services/messaging/email.service.test.ts
@@ -4,6 +4,7 @@ import { MessageRejected, SendingPausedException, TooManyRequestsException } fro
import { DateTime } from 'luxon'
import { register } from 'prom-client'
+import { FixtureHogFlowBuilder } from '~/cdp/_tests/builders/hogflow.builder'
import { createExampleInvocation, insertIntegration } from '~/cdp/_tests/fixtures'
import { EmailStatusOutput } from '~/cdp/outputs/outputs'
import {
@@ -144,10 +145,12 @@ describe('EmailService', () => {
let service: EmailService
let hub: Hub
let team: Team
+ let workflowsActivationReporter: { report: jest.Mock }
beforeEach(async () => {
hub = await createHub({})
team = (await createTestTeamFixture(hub.postgres)).team
integrationIdBase = team.id
+ workflowsActivationReporter = { report: jest.fn() }
service = new EmailService(
{
sesAccessKeyId: hub.SES_ACCESS_KEY_ID,
@@ -164,7 +167,8 @@ describe('EmailService', () => {
new EmailTrackingCodeSigner(hub.ENCRYPTION_SALT_KEYS, hub.CDP_EMAIL_TRACKING_URL),
new EmailSuppressionService(hub.postgres, emailSuppressionConfigFromEnv()),
new RecipientsManagerService(hub.postgres),
- null
+ null,
+ workflowsActivationReporter
)
mockFetch.mockClear()
})
@@ -189,7 +193,8 @@ describe('EmailService', () => {
new EmailTrackingCodeSigner(hub.ENCRYPTION_SALT_KEYS, hub.CDP_EMAIL_TRACKING_URL),
new EmailSuppressionService(hub.postgres, emailSuppressionConfigFromEnv()),
new RecipientsManagerService(hub.postgres),
- null
+ null,
+ { report: jest.fn() }
)
expect(serviceWithoutSES.sesV2Client).toBeNull()
@@ -286,13 +291,32 @@ describe('EmailService', () => {
`"Email integration not found. The sender configured for this step no longer exists — select a new sender in the workflow's email step."`
)
})
- it('should validate if the email domain is not verified', async () => {
- invocation.queueParameters = createEmailParams({
- from: { integrationId: 2 },
- })
- const result = await service.executeSendEmail(invocation)
- expect(result.error).toMatchInlineSnapshot(`"The selected email integration domain is not verified"`)
- })
+ it.each([
+ ['a workflow send', true, false, 1],
+ ['a workflow test send', true, true, 0],
+ ['a hog function send', false, false, 0],
+ ])(
+ 'should fail on an unverified email domain and report it only for a real workflow send: %s',
+ async (_name, fromWorkflow, isTest, reports) => {
+ invocation.queueParameters = createEmailParams({
+ from: { integrationId: 2 },
+ })
+ const sentInvocation = fromWorkflow
+ ? { ...invocation, hogFlow: new FixtureHogFlowBuilder().withTeamId(team.id).build() }
+ : invocation
+ const result = await service.executeSendEmail(sentInvocation, isTest)
+ expect(result.error).toMatchInlineSnapshot(
+ `"The selected email integration domain is not verified"`
+ )
+ expect(workflowsActivationReporter.report.mock.calls).toEqual(
+ Array(reports).fill([
+ team.id,
+ 'workflows send failed',
+ { reason: 'unverified_domain', channel: 'email', workflow_id: invocation.functionId },
+ ])
+ )
+ }
+ )
it('should send identical from and feedback forwarding args', async () => {
// This test is important for spam classification - feedback forwarding email MUST match from email
invocation.queueParameters = createEmailParams({
@@ -633,6 +657,7 @@ describe('EmailService', () => {
new EmailSuppressionService(hub.postgres, emailSuppressionConfigFromEnv()),
new RecipientsManagerService(hub.postgres),
null,
+ { report: jest.fn() },
undefined,
{ claimOrReserve } as unknown as RateLimiterService
)
@@ -805,6 +830,7 @@ describe('EmailService', () => {
new EmailSuppressionService(hub.postgres, emailSuppressionConfigFromEnv()),
new RecipientsManagerService(hub.postgres),
null,
+ { report: jest.fn() },
undefined,
new RateLimiterService(redis, { name: 'workflow-email-backlog-test' })
)
@@ -876,6 +902,7 @@ describe('EmailService', () => {
new EmailSuppressionService(hub.postgres, emailSuppressionConfigFromEnv()),
new RecipientsManagerService(hub.postgres),
null,
+ { report: jest.fn() },
undefined,
null,
{ claimAllOrNothingPair } as unknown as RateLimiterService
@@ -1007,6 +1034,7 @@ describe('EmailService', () => {
new EmailSuppressionService(hub.postgres, emailSuppressionConfigFromEnv()),
new RecipientsManagerService(hub.postgres),
null,
+ { report: jest.fn() },
undefined,
null,
limiter
@@ -1398,6 +1426,33 @@ describe('EmailService', () => {
expect(invocation.state.vmState?.stack).toEqual([{ success: false }])
})
+ it.each([
+ ['a workflow send', true, false, 1],
+ ['a workflow test send', true, true, 0],
+ ['a hog function send', false, false, 0],
+ ])(
+ 'reports a missing recipient as a blocked send only for a real workflow send: %s',
+ async (_name, fromWorkflow, isTest, reports) => {
+ invocation.queueParameters = createEmailParams({
+ from: { integrationId: 1 },
+ to: { email: '', name: 'Test User' },
+ })
+ const sentInvocation = fromWorkflow
+ ? { ...invocation, hogFlow: new FixtureHogFlowBuilder().withTeamId(team.id).build() }
+ : invocation
+
+ await service.executeSendEmail(sentInvocation, isTest)
+
+ expect(workflowsActivationReporter.report.mock.calls).toEqual(
+ Array(reports).fill([
+ team.id,
+ 'workflows send blocked',
+ { reason: 'missing_recipient', channel: 'email', workflow_id: invocation.functionId },
+ ])
+ )
+ }
+ )
+
describe('SES tenant attribution', () => {
it('attributes the send to the team tenant', async () => {
sendEmailSpy.mockResolvedValue({ MessageId: 'test-message-id' })
diff --git a/nodejs/src/cdp/services/messaging/email.service.ts b/nodejs/src/cdp/services/messaging/email.service.ts
index c67d2f7e592b..7bb4a99cce01 100644
--- a/nodejs/src/cdp/services/messaging/email.service.ts
+++ b/nodejs/src/cdp/services/messaging/email.service.ts
@@ -20,6 +20,7 @@ import { TRANSACTIONAL_EMAIL_EVENT } from '../hogflows/hogflow-sensitivity'
import { IntegrationManagerService } from '../managers/integration-manager.service'
import { RecipientManagerRecipient, RecipientsManagerService } from '../managers/recipients-manager.service'
import { TeamWorkflowsConfigService } from '../managers/team-workflows-config.service'
+import { WorkflowsActivationReporter } from '../monitoring/workflows-activation-reporter'
import { RateLimiterService } from '../rate-limiter/rate-limiter.service'
import { getEmailQueuePriorityClass } from './email-priority'
import { selectEmailSenderIntegrationId } from './email-sender-selection'
@@ -153,6 +154,16 @@ export class SESThrottleError extends Error {
}
}
+class UnverifiedEmailDomainError extends Error {
+ constructor() {
+ super('The selected email integration domain is not verified')
+ this.name = 'UnverifiedEmailDomainError'
+ }
+}
+
+const isRealWorkflowSend = (invocation: CyclotronJobInvocationHogFunction, isTest: boolean): boolean =>
+ !isTest && 'hogFlow' in invocation
+
function pickThrottleRetryDelayMs(): number {
// Constant 500–1000ms jitter is plenty: the local Valkey bucket already
// gates re-dequeue at the configured refill rate, so a quick retry will
@@ -424,6 +435,7 @@ export class EmailService {
// Required, so that a caller which builds the service without it fails to compile. Pass null
// only where no email status is recorded.
private emailStatusService: EmailStatusService | null,
+ private workflowsActivationReporter: Pick<WorkflowsActivationReporter, 'report'>,
private messageAssetsService?: MessageAssetsService,
private workflowEmailRateLimiter: RateLimiterService | null = null,
private teamEmailRateLimiter: RateLimiterService | null = null
@@ -577,6 +589,13 @@ export class EmailService {
})
}
this.emailStatusService?.queueSkippedSend(invocation, 'missing_recipient', isTest)
+ if (isRealWorkflowSend(invocation, isTest)) {
+ void this.workflowsActivationReporter.report(invocation.teamId, 'workflows send blocked', {
+ reason: 'missing_recipient',
+ channel: 'email',
+ workflow_id: invocation.functionId,
+ })
+ }
result.skipped = true
result.invocation.state.vmState?.stack.push({ success: false })
return result
@@ -777,6 +796,13 @@ export class EmailService {
result.error = error.message
result.emailSendResult = { status: 'failed', reason: error.message }
result.finished = true
+ if (error instanceof UnverifiedEmailDomainError && isRealWorkflowSend(invocation, isTest)) {
+ void this.workflowsActivationReporter.report(invocation.teamId, 'workflows send failed', {
+ reason: 'unverified_domain',
+ channel: 'email',
+ workflow_id: invocation.functionId,
+ })
+ }
}
}
@@ -1059,7 +1085,7 @@ export class EmailService {
addLog: ReturnType<typeof createAddLogFunction>
): { email: string; name: string } {
if (!integration.config.verified) {
- throw new Error('The selected email integration domain is not verified')
+ throw new UnverifiedEmailDomainError()
}
if (!integration.config.email || !integration.config.name) {
diff --git a/nodejs/src/cdp/services/monitoring/workflows-activation-reporter.test.ts b/nodejs/src/cdp/services/monitoring/workflows-activation-reporter.test.ts
new file mode 100644
index 000000000000..a2c7b8f9cc94
--- /dev/null
+++ b/nodejs/src/cdp/services/monitoring/workflows-activation-reporter.test.ts
@@ -0,0 +1,103 @@
+import { Team } from '~/types'
+
+import { WorkflowsActivationReporter } from './workflows-activation-reporter'
+
+describe('WorkflowsActivationReporter', () => {
+ const team = { id: 2, uuid: 'team-uuid', organization_id: 'org-id' } as Team
+ const otherTeam = { id: 3, uuid: 'other-team-uuid', organization_id: 'org-id' } as Team
+ let getTeam: jest.Mock
+ let capture: jest.Mock
+ let reporter: WorkflowsActivationReporter
+
+ beforeEach(() => {
+ jest.useFakeTimers({ now: new Date('2026-10-03T10:00:00Z') })
+ getTeam = jest.fn((teamId: number) => Promise.resolve([team, otherTeam].find((t) => t.id === teamId) ?? null))
+ capture = jest.fn()
+ reporter = new WorkflowsActivationReporter({ getTeam }, capture)
+ })
+
+ afterEach(() => {
+ jest.useRealTimers()
+ })
+
+ it('captures each team and reason once per hour', async () => {
+ await reporter.report(2, 'workflows send failed', {
+ reason: 'unverified_domain',
+ channel: 'email',
+ workflow_id: 'flow-1',
+ })
+ await reporter.report(2, 'workflows send failed', {
+ reason: 'unverified_domain',
+ channel: 'email',
+ workflow_id: 'flow-2',
+ })
+ await reporter.report(3, 'workflows send failed', {
+ reason: 'unverified_domain',
+ channel: 'email',
+ workflow_id: 'flow-9',
+ })
+ await reporter.report(2, 'workflows send blocked', {
+ reason: 'missing_recipient',
+ channel: 'email',
+ workflow_id: 'flow-1',
+ })
+ jest.advanceTimersByTime(60 * 60 * 1000)
+ await reporter.report(2, 'workflows send failed', {
+ reason: 'unverified_domain',
+ channel: 'email',
+ workflow_id: 'flow-3',
+ })
+
+ expect(capture.mock.calls).toEqual([
+ [team, 'workflows send failed', { reason: 'unverified_domain', channel: 'email', workflow_id: 'flow-1' }],
+ [
+ otherTeam,
+ 'workflows send failed',
+ { reason: 'unverified_domain', channel: 'email', workflow_id: 'flow-9' },
+ ],
+ [team, 'workflows send blocked', { reason: 'missing_recipient', channel: 'email', workflow_id: 'flow-1' }],
+ [team, 'workflows send failed', { reason: 'unverified_domain', channel: 'email', workflow_id: 'flow-3' }],
+ ])
+ })
+
+ it('skips teams it cannot find', async () => {
+ await reporter.report(404, 'workflows send blocked', {
+ reason: 'quota_limited',
+ channel: 'email',
+ workflow_id: 'flow-1',
+ })
+
+ expect(capture).not.toHaveBeenCalled()
+ })
+
+ it.each([
+ ['a failed team lookup', () => getTeam.mockRejectedValueOnce(new Error('postgres is down'))],
+ ['a missing team', () => getTeam.mockResolvedValueOnce(null)],
+ [
+ 'a failed capture',
+ () =>
+ capture.mockImplementationOnce(() => {
+ throw new Error('capture is down')
+ }),
+ ],
+ ])('reports again after %s', async (_name, failOnce) => {
+ failOnce()
+
+ await reporter.report(2, 'workflows send blocked', {
+ reason: 'quota_limited',
+ channel: 'email',
+ workflow_id: 'flow-1',
+ })
+ await reporter.report(2, 'workflows send blocked', {
+ reason: 'quota_limited',
+ channel: 'email',
+ workflow_id: 'flow-2',
+ })
+
+ expect(capture).toHaveBeenLastCalledWith(team, 'workflows send blocked', {
+ reason: 'quota_limited',
+ channel: 'email',
+ workflow_id: 'flow-2',
+ })
+ })
+})
diff --git a/nodejs/src/cdp/services/monitoring/workflows-activation-reporter.ts b/nodejs/src/cdp/services/monitoring/workflows-activation-reporter.ts
new file mode 100644
index 000000000000..c43881115e9f
--- /dev/null
+++ b/nodejs/src/cdp/services/monitoring/workflows-activation-reporter.ts
@@ -0,0 +1,47 @@
+import { LRUCache } from 'lru-cache'
+
+import { logger } from '~/common/utils/logger'
+import { captureTeamEvent } from '~/common/utils/posthog'
+import { Team } from '~/types'
+
+type WorkflowsActivationEvents = {
+ 'workflows send blocked': { workflow_id: string; channel: 'email'; reason: 'quota_limited' | 'missing_recipient' }
+ 'workflows send failed': { workflow_id: string; channel: 'email'; reason: 'unverified_domain' }
+}
+
+const REPORT_INTERVAL_MS = 60 * 60 * 1000
+
+export class WorkflowsActivationReporter {
+ private lastReportedAt = new LRUCache<string, number>({ max: 50_000 })
+
+ constructor(
+ private teamManager: { getTeam(teamId: number): Promise<Team | null> },
+ private capture: typeof captureTeamEvent = captureTeamEvent
+ ) {}
+
+ public async report<Event extends keyof WorkflowsActivationEvents>(
+ teamId: number,
+ event: Event,
+ properties: WorkflowsActivationEvents[Event]
+ ): Promise<void> {
+ const key = `${teamId}:${event}:${properties.reason}`
+ const now = Date.now()
+ const lastReportedAt = this.lastReportedAt.get(key)
+ if (lastReportedAt !== undefined && now - lastReportedAt < REPORT_INTERVAL_MS) {
+ return
+ }
+ this.lastReportedAt.set(key, now)
+
+ try {
+ const team = await this.teamManager.getTeam(teamId)
+ if (!team) {
+ this.lastReportedAt.delete(key)
+ return
+ }
+ this.capture(team, event, properties)
+ } catch (error) {
+ this.lastReportedAt.delete(key)
+ logger.warn('Failed to report a workflows activation event', { teamId, event, error })
+ }
+ }
+}
diff --git a/nodejs/src/cdp/templates/test/test-helpers.ts b/nodejs/src/cdp/templates/test/test-helpers.ts
index 6fb026c1b3b1..87e795a81013 100644
--- a/nodejs/src/cdp/templates/test/test-helpers.ts
+++ b/nodejs/src/cdp/templates/test/test-helpers.ts
@@ -221,7 +221,8 @@ export class TemplateTester {
new EmailTrackingCodeSigner(config.ENCRYPTION_SALT_KEYS, config.CDP_EMAIL_TRACKING_URL),
undefined as any,
undefined as any,
- null
+ null,
+ { report: () => Promise.resolve() }
)
return new HogExecutorAsyncService(
new HogExecutorService(