:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(workflows): report blocked and failed workflow sends (#127135)
Michael Reichenbach5fd7b11654da
diff --git a/nodejs/src/cdp/cdp-services.ts b/nodejs/src/cdp/cdp-services.tsindex c3bda80b7084..79187db4a904 100644--- a/nodejs/src/cdp/cdp-services.ts+++ b/nodejs/src/cdp/cdp-services.ts@@ -47,6 +47,7 @@ import { RecipientTokensService } from './services/messaging/recipient-tokens.seimport { HogFunctionMonitoringService } from './services/monitoring/hog-function-monitoring.service'import { HogInvocationResultsService } from './services/monitoring/hog-invocation-results.service'import { HogWatcherService } from './services/monitoring/hog-watcher.service'+import { WorkflowsActivationReporter } from './services/monitoring/workflows-activation-reporter'import { NativeDestinationExecutorService } from './services/native-destination-executor.service'import { RateLimiterService } from './services/rate-limiter/rate-limiter.service'import { SegmentDestinationExecutorService } from './services/segment-destination-executor.service'@@ -106,6 +107,7 @@ export interface CdpCoreServices {cohortMembershipRepository: CohortMembershipRepositoryhogFlowExecutor: HogFlowExecutorServicehogFunctionMonitoringService: HogFunctionMonitoringService+ workflowsActivationReporter: WorkflowsActivationReportercdpUsageReporter: CdpUsageReporterServicecapturedEventsService: CapturedEventsService/** Per-invocation lifecycle row producer for the new runs/invocations UI + rerun path. */@@ -453,6 +455,7 @@ export function createCdpCoreServices(})const recipientsManager = new RecipientsManagerService(deps.postgres)const personsManager = new PersonsManagerService(deps.teamManager, deps.personRepository, config.SITE_URL)+ const workflowsActivationReporter = new WorkflowsActivationReporter(deps.teamManager)// Per-workflow send pacing rides the SES Valkey pool, which is only populated on pods that// execute email actions — exactly where the limit is enforced. Null elsewhere disables it.const workflowEmailRateLimiter = deps.emailValidationValkey@@ -483,6 +486,7 @@ export function createCdpCoreServices(emailSuppressionService,recipientsManager,emailStatusService,+ workflowsActivationReporter,messageAssetsService,workflowEmailRateLimiter,teamEmailRateLimiter@@ -624,6 +628,7 @@ export function createCdpCoreServices(cohortMembershipRepository,hogFlowExecutor,hogFunctionMonitoringService,+ workflowsActivationReporter,cdpUsageReporter,capturedEventsService,hogInvocationResultsService,diff --git a/nodejs/src/cdp/consumers/cdp-base.consumer.ts b/nodejs/src/cdp/consumers/cdp-base.consumer.tsindex c27500666b02..62ed7859cec3 100644--- a/nodejs/src/cdp/consumers/cdp-base.consumer.ts+++ b/nodejs/src/cdp/consumers/cdp-base.consumer.ts@@ -31,6 +31,7 @@ import { RecipientPreferencesService } from '../services/messaging/recipient-preimport { HogFunctionMonitoringService } from '../services/monitoring/hog-function-monitoring.service'import { HogMaskerService } from '../services/monitoring/hog-masker.service'import { HogWatcherService } from '../services/monitoring/hog-watcher.service'+import { WorkflowsActivationReporter } from '../services/monitoring/workflows-activation-reporter'import { NativeDestinationExecutorService } from '../services/native-destination-executor.service'import { SegmentDestinationExecutorService } from '../services/segment-destination-executor.service'import { CdpUsageReporterService } from '../services/usage/cdp-usage-reporter.service'@@ -73,6 +74,7 @@ export abstract class CdpConsumerBase<TConfig extends CdpConsumerBaseConfig = CdemailService: EmailServicehogFunctionMonitoringService: HogFunctionMonitoringService+ workflowsActivationReporter: WorkflowsActivationReportercdpUsageReporter: CdpUsageReporterServiceinvocationResultsService: InvocationResultsServicenativeDestinationExecutorService: NativeDestinationExecutorService@@ -105,6 +107,7 @@ export abstract class CdpConsumerBase<TConfig extends CdpConsumerBaseConfig = Cdthis.hogFlowExecutor = services.hogFlowExecutorthis.emailService = services.emailServicethis.hogFunctionMonitoringService = services.hogFunctionMonitoringService+ this.workflowsActivationReporter = services.workflowsActivationReporterthis.cdpUsageReporter = services.cdpUsageReporterthis.invocationResultsService = services.invocationResultsServicethis.nativeDestinationExecutorService = services.nativeDestinationExecutorServicediff --git a/nodejs/src/cdp/consumers/cdp-cohort-membership-trigger.consumer.ts b/nodejs/src/cdp/consumers/cdp-cohort-membership-trigger.consumer.tsindex a52e697fe750..5538e8df5b1a 100644--- a/nodejs/src/cdp/consumers/cdp-cohort-membership-trigger.consumer.ts+++ b/nodejs/src/cdp/consumers/cdp-cohort-membership-trigger.consumer.ts@@ -102,6 +102,7 @@ export class CdpCohortMembershipTriggerConsumer extends CdpConsumerBase {hogWatcherMirror: this.hogWatcherMirror,hogMasker: this.hogMasker,hogFunctionMonitoringService: this.hogFunctionMonitoringService,+ workflowsActivationReporter: this.workflowsActivationReporter,quotaLimiting: deps.quotaLimiting,redis: this.redis,valkeyShadow: this.valkeyShadow,diff --git a/nodejs/src/cdp/consumers/cdp-data-warehouse-events.consumer.ts b/nodejs/src/cdp/consumers/cdp-data-warehouse-events.consumer.tsindex 2fe93daf3416..66016324e615 100644--- a/nodejs/src/cdp/consumers/cdp-data-warehouse-events.consumer.ts+++ b/nodejs/src/cdp/consumers/cdp-data-warehouse-events.consumer.ts@@ -73,6 +73,7 @@ export class CdpDatawarehouseEventsConsumer extends CdpConsumerBase {hogWatcherMirror: this.hogWatcherMirror,hogMasker: this.hogMasker,hogFunctionMonitoringService: this.hogFunctionMonitoringService,+ workflowsActivationReporter: this.workflowsActivationReporter,quotaLimiting: deps.quotaLimiting,redis: this.redis,valkeyShadow: this.valkeyShadow,diff --git a/nodejs/src/cdp/consumers/cdp-dlq-replay.consumer.ts b/nodejs/src/cdp/consumers/cdp-dlq-replay.consumer.tsindex 2f8a787c6a73..2b7bdcfc5b83 100644--- a/nodejs/src/cdp/consumers/cdp-dlq-replay.consumer.ts+++ b/nodejs/src/cdp/consumers/cdp-dlq-replay.consumer.ts@@ -151,6 +151,7 @@ export class CdpDlqReplayConsumer extends CdpConsumerBase<PluginsServerConfig> {hogWatcherMirror: this.hogWatcherMirror,hogMasker: this.hogMasker,hogFunctionMonitoringService: this.hogFunctionMonitoringService,+ workflowsActivationReporter: this.workflowsActivationReporter,quotaLimiting: deps.quotaLimiting,redis: this.redis,valkeyShadow: this.valkeyShadow,diff --git a/nodejs/src/cdp/consumers/cdp-events.consumer.ts b/nodejs/src/cdp/consumers/cdp-events.consumer.tsindex 922b44dd7b49..a6cbbfaf04fc 100644--- a/nodejs/src/cdp/consumers/cdp-events.consumer.ts+++ b/nodejs/src/cdp/consumers/cdp-events.consumer.ts@@ -71,6 +71,7 @@ export class CdpEventsConsumer<hogWatcherMirror: this.hogWatcherMirror,hogMasker: this.hogMasker,hogFunctionMonitoringService: this.hogFunctionMonitoringService,+ workflowsActivationReporter: this.workflowsActivationReporter,quotaLimiting: deps.quotaLimiting,redis: this.redis,valkeyShadow: this.valkeyShadow,diff --git a/nodejs/src/cdp/consumers/cdp-internal-event.consumer.ts b/nodejs/src/cdp/consumers/cdp-internal-event.consumer.tsindex 22c703c741a3..678b5c6f6f67 100644--- a/nodejs/src/cdp/consumers/cdp-internal-event.consumer.ts+++ b/nodejs/src/cdp/consumers/cdp-internal-event.consumer.ts@@ -78,6 +78,7 @@ export class CdpInternalEventsConsumer extends CdpConsumerBase {hogWatcherMirror: this.hogWatcherMirror,hogMasker: this.hogMasker,hogFunctionMonitoringService: this.hogFunctionMonitoringService,+ workflowsActivationReporter: this.workflowsActivationReporter,quotaLimiting: deps.quotaLimiting,redis: this.redis,valkeyShadow: this.valkeyShadow,diff --git a/nodejs/src/cdp/services/hog-executor.service.test.ts b/nodejs/src/cdp/services/hog-executor.service.test.tsindex d110b6434e04..41bec369ca49 100644--- a/nodejs/src/cdp/services/hog-executor.service.test.ts+++ b/nodejs/src/cdp/services/hog-executor.service.test.ts@@ -89,7 +89,8 @@ describe('Hog Executor', () => {new EmailTrackingCodeSigner(hub.ENCRYPTION_SALT_KEYS, hub.CDP_EMAIL_TRACKING_URL),new EmailSuppressionService(hub.postgres, emailSuppressionConfigFromEnv()),new RecipientsManagerService(hub.postgres),- null+ null,+ { report: jest.fn() })const recipientTokensService = new RecipientTokensService(hub.ENCRYPTION_SALT_KEYS, hub.SITE_URL)return new HogExecutorAsyncService(diff --git a/nodejs/src/cdp/services/hog-flow-invocation-pipeline.service.test.ts b/nodejs/src/cdp/services/hog-flow-invocation-pipeline.service.test.tsindex 0e3a67ebe439..8220ecea846a 100644--- a/nodejs/src/cdp/services/hog-flow-invocation-pipeline.service.test.ts+++ b/nodejs/src/cdp/services/hog-flow-invocation-pipeline.service.test.ts@@ -49,6 +49,7 @@ describe('HogFlowInvocationPipeline', () => {let hogMasker: jest.Mocked<HogMaskerService>let hogFunctionMonitoringService: jest.Mocked<HogFunctionMonitoringService>let quotaLimiting: jest.Mocked<QuotaLimiting>+ let workflowsActivationReporter: { report: jest.Mock }let pipeline: HogFlowInvocationPipelinelet rateLimitGroupedMock: jest.Mock@@ -83,6 +84,8 @@ describe('HogFlowInvocationPipeline', () => {isTeamQuotaLimited: jest.fn().mockResolvedValue(false),} as unknown as jest.Mocked<QuotaLimiting>+ workflowsActivationReporter = { report: jest.fn() }+pipeline = new HogFlowInvocationPipeline(config, {hogFlowManager,hogFlowExecutor,@@ -90,6 +93,7 @@ describe('HogFlowInvocationPipeline', () => {hogWatcherMirror: hogWatcher,hogMasker,hogFunctionMonitoringService,+ workflowsActivationReporter,quotaLimiting,redis: {} as any,valkeyShadow: { writer: {} as any, reader: {} as any },@@ -175,6 +179,11 @@ describe('HogFlowInvocationPipeline', () => {const result = await pipeline.buildInvocations([makeGlobals()])expect(result).toEqual([])+ expect(workflowsActivationReporter.report).toHaveBeenCalledWith(inv.teamId, 'workflows send blocked', {+ reason: 'quota_limited',+ channel: 'email',+ workflow_id: inv.functionId,+ })})it('drops invocations for disabled hogflows', async () => {diff --git a/nodejs/src/cdp/services/hog-flow-invocation-pipeline.service.ts b/nodejs/src/cdp/services/hog-flow-invocation-pipeline.service.tsindex 6d91fb01172d..bc016ff9f800 100644--- a/nodejs/src/cdp/services/hog-flow-invocation-pipeline.service.ts+++ b/nodejs/src/cdp/services/hog-flow-invocation-pipeline.service.ts@@ -25,6 +25,7 @@ import { shouldBlockHogFlowDueToQuota } from './hogflows/hogflow-quota-limiting'import { HogFunctionMonitoringService } from './monitoring/hog-function-monitoring.service'import { HogMaskerService } from './monitoring/hog-masker.service'import { HogWatcherService, HogWatcherState, sameWatcherStates } from './monitoring/hog-watcher.service'+import { WorkflowsActivationReporter } from './monitoring/workflows-activation-reporter'export interface HogFlowInvocationPipelineConfig {CDP_RATE_LIMITER_BUCKET_SIZE: number@@ -39,6 +40,7 @@ export interface HogFlowInvocationPipelineDeps {hogWatcherMirror: HogWatcherServicehogMasker: HogMaskerServicehogFunctionMonitoringService: HogFunctionMonitoringService+ workflowsActivationReporter: Pick<WorkflowsActivationReporter, 'report'>quotaLimiting: QuotaLimitingredis: RedisV2valkeyShadow: CdpValkeyShadowPools@@ -210,6 +212,7 @@ export class HogFlowInvocationPipeline {const isQuotaLimited = await shouldBlockHogFlowDueToQuota(item, {quotaLimiting: this.deps.quotaLimiting,hogFunctionMonitoringService: this.deps.hogFunctionMonitoringService,+ workflowsActivationReporter: this.deps.workflowsActivationReporter,})if (isQuotaLimited) {diff --git a/nodejs/src/cdp/services/hogflows/actions/hog_function.test.ts b/nodejs/src/cdp/services/hogflows/actions/hog_function.test.tsindex bdb88d9c8caf..6ac510fb1bbd 100644--- a/nodejs/src/cdp/services/hogflows/actions/hog_function.test.ts+++ b/nodejs/src/cdp/services/hogflows/actions/hog_function.test.ts@@ -93,7 +93,8 @@ describe('HogFunctionHandler', () => {new EmailTrackingCodeSigner(hub.ENCRYPTION_SALT_KEYS, hub.CDP_EMAIL_TRACKING_URL),new EmailSuppressionService(hub.postgres, emailSuppressionConfigFromEnv()),new RecipientsManagerService(hub.postgres),- null+ null,+ { report: jest.fn() })mockHogFunctionExecutor = new HogExecutorAsyncService(new HogExecutorService({ executionTimeoutMs: hub.CDP_WATCHER_HOG_COST_TIMING_UPPER_MS }, hogInputsService),diff --git a/nodejs/src/cdp/services/hogflows/hogflow-executor.service.test.ts b/nodejs/src/cdp/services/hogflows/hogflow-executor.service.test.tsindex c632c352bf77..49c0688590be 100644--- a/nodejs/src/cdp/services/hogflows/hogflow-executor.service.test.ts+++ b/nodejs/src/cdp/services/hogflows/hogflow-executor.service.test.ts@@ -100,7 +100,8 @@ describe('Hogflow Executor', () => {new EmailTrackingCodeSigner(hub.ENCRYPTION_SALT_KEYS, hub.CDP_EMAIL_TRACKING_URL),emailSuppressionService,new RecipientsManagerService(hub.postgres),- null+ null,+ { report: jest.fn() })const recipientTokensService = new RecipientTokensService(hub.ENCRYPTION_SALT_KEYS, hub.SITE_URL)const hogExecutor = new HogExecutorAsyncService(diff --git a/nodejs/src/cdp/services/hogflows/hogflow-quota-limiting.test.ts b/nodejs/src/cdp/services/hogflows/hogflow-quota-limiting.test.tsindex ed38724827a8..ac02ae9d8849 100644--- a/nodejs/src/cdp/services/hogflows/hogflow-quota-limiting.test.ts+++ b/nodejs/src/cdp/services/hogflows/hogflow-quota-limiting.test.ts@@ -216,6 +216,7 @@ describe('HogFlow Quota Limiting', () => {describe('shouldBlockHogFlowDueToQuota', () => {let mockHogFunctionMonitoringService: jest.Mocked<HogFunctionMonitoringService>+ let mockWorkflowsActivationReporter: { report: jest.Mock }const teamId = 123const baseItem: CyclotronJobInvocationHogFlow = {teamId,@@ -238,6 +239,7 @@ describe('HogFlow Quota Limiting', () => {mockHogFunctionMonitoringService = {queueAppMetric: jest.fn(),} as any+ mockWorkflowsActivationReporter = { report: jest.fn() }// Reset the counter spyjest.spyOn(counterHogFlowQuotaLimited, 'labels').mockReturnValue({inc: jest.fn(),@@ -258,29 +260,58 @@ describe('HogFlow Quota Limiting', () => {const result = await shouldBlockHogFlowDueToQuota(item, {quotaLimiting: mockQuotaLimiting,hogFunctionMonitoringService: mockHogFunctionMonitoringService,+ workflowsActivationReporter: mockWorkflowsActivationReporter,})expect(result).toBe(false)expect(mockHogFunctionMonitoringService.queueAppMetric).not.toHaveBeenCalled()expect(counterHogFlowQuotaLimited.labels).not.toHaveBeenCalled()+ expect(mockWorkflowsActivationReporter.report).not.toHaveBeenCalled()})- it('should block invocation and emit metrics when quota limited', async () => {+ it.each([+ [+ 'an email send on the email quota',+ 'workflow_emails',+ ['function_email'],+ [+ [+ teamId,+ 'workflows send blocked',+ { reason: 'quota_limited', channel: 'email', workflow_id: 'test-flow-id' },+ ],+ ],+ ],+ ['a webhook on the destination quota', 'workflow_destinations_dispatched', ['function'], []],+ [+ 'an email send next to a webhook on the destination quota',+ 'workflow_destinations_dispatched',+ ['function_email', 'function'],+ [+ [+ teamId,+ 'workflows send blocked',+ { reason: 'quota_limited', channel: 'email', workflow_id: 'test-flow-id' },+ ],+ ],+ ],+ ] as const)('should block %s and emit metrics', async (_name, limitedResource, actionTypes, reports) => {mockQuotaLimiting.isTeamQuotaLimited.mockImplementation((_teamId, resource) => {- return Promise.resolve(resource === 'workflow_emails')+ return Promise.resolve(resource === limitedResource)})const item: CyclotronJobInvocationHogFlow = {...baseItem,hogFlow: {...baseItem.hogFlow,- billable_action_types: ['function_email'],+ billable_action_types: [...actionTypes],},}const result = await shouldBlockHogFlowDueToQuota(item, {quotaLimiting: mockQuotaLimiting,hogFunctionMonitoringService: mockHogFunctionMonitoringService,+ workflowsActivationReporter: mockWorkflowsActivationReporter,})expect(result).toBe(true)@@ -296,6 +327,36 @@ describe('HogFlow Quota Limiting', () => {},'hog_flow')+ expect(mockWorkflowsActivationReporter.report.mock.calls).toEqual(reports)+ })++ it('should report a blocked email send when the destination quota stops transactional emails next to a webhook', async () => {+ mockQuotaLimiting.isTeamQuotaLimited.mockResolvedValue(true)++ const item: CyclotronJobInvocationHogFlow = {+ ...baseItem,+ hogFlow: {+ ...baseItem.hogFlow,+ actions: [+ { type: 'function_email', config: { message_category_type: 'transactional' } },+ { type: 'function', config: {} },+ ],+ billable_action_types: ['function_email', 'function'],+ } as unknown as HogFlow,+ }++ const result = await shouldBlockHogFlowDueToQuota(item, {+ quotaLimiting: mockQuotaLimiting,+ hogFunctionMonitoringService: mockHogFunctionMonitoringService,+ workflowsActivationReporter: mockWorkflowsActivationReporter,+ })++ expect(result).toBe(true)+ expect(mockWorkflowsActivationReporter.report).toHaveBeenCalledWith(teamId, 'workflows send blocked', {+ reason: 'quota_limited',+ channel: 'email',+ workflow_id: 'test-flow-id',+ })})it('should handle workflow with no billable actions', async () => {@@ -312,6 +373,7 @@ describe('HogFlow Quota Limiting', () => {const result = await shouldBlockHogFlowDueToQuota(item, {quotaLimiting: mockQuotaLimiting,hogFunctionMonitoringService: mockHogFunctionMonitoringService,+ workflowsActivationReporter: mockWorkflowsActivationReporter,})expect(result).toBe(false)diff --git a/nodejs/src/cdp/services/hogflows/hogflow-quota-limiting.ts b/nodejs/src/cdp/services/hogflows/hogflow-quota-limiting.tsindex 849173a89e80..f04e35eb0774 100644--- a/nodejs/src/cdp/services/hogflows/hogflow-quota-limiting.ts+++ b/nodejs/src/cdp/services/hogflows/hogflow-quota-limiting.ts@@ -5,6 +5,7 @@ import { HogFlow, HogFlowAction } from '~/cdp/schema/hogflow'import { QuotaLimiting } from '../../../common/services/quota-limiting.service'import { CyclotronJobInvocationHogFlow } from '../../types'import { HogFunctionMonitoringService } from '../monitoring/hog-function-monitoring.service'+import { WorkflowsActivationReporter } from '../monitoring/workflows-activation-reporter'export const counterHogFlowQuotaLimited = new Counter({name: 'cdp_hog_flow_quota_limited',@@ -73,6 +74,11 @@ export async function checkHogFlowQuotaLimits(export interface HogFlowQuotaLimitingContext {quotaLimiting: QuotaLimitinghogFunctionMonitoringService: HogFunctionMonitoringService+ workflowsActivationReporter: Pick<WorkflowsActivationReporter, 'report'>+}++function sendsEmail(hogFlow: HogFlow): boolean {+ return Array.isArray(hogFlow.billable_action_types) && hogFlow.billable_action_types.includes('function_email')}/**@@ -99,6 +105,13 @@ export async function shouldBlockHogFlowDueToQuota(},'hog_flow')+ if (sendsEmail(item.hogFlow)) {+ void context.workflowsActivationReporter.report(item.teamId, 'workflows send blocked', {+ reason: 'quota_limited',+ channel: 'email',+ workflow_id: item.functionId,+ })+ }return true}diff --git a/nodejs/src/cdp/services/messaging/email.service.test.ts b/nodejs/src/cdp/services/messaging/email.service.test.tsindex d42d6399c11a..836d3ca98b86 100644--- a/nodejs/src/cdp/services/messaging/email.service.test.ts+++ b/nodejs/src/cdp/services/messaging/email.service.test.ts@@ -4,6 +4,7 @@ import { MessageRejected, SendingPausedException, TooManyRequestsException } froimport { DateTime } from 'luxon'import { register } from 'prom-client'+import { FixtureHogFlowBuilder } from '~/cdp/_tests/builders/hogflow.builder'import { createExampleInvocation, insertIntegration } from '~/cdp/_tests/fixtures'import { EmailStatusOutput } from '~/cdp/outputs/outputs'import {@@ -144,10 +145,12 @@ describe('EmailService', () => {let service: EmailServicelet hub: Hublet team: Team+ let workflowsActivationReporter: { report: jest.Mock }beforeEach(async () => {hub = await createHub({})team = (await createTestTeamFixture(hub.postgres)).teamintegrationIdBase = team.id+ workflowsActivationReporter = { report: jest.fn() }service = new EmailService({sesAccessKeyId: hub.SES_ACCESS_KEY_ID,@@ -164,7 +167,8 @@ describe('EmailService', () => {new EmailTrackingCodeSigner(hub.ENCRYPTION_SALT_KEYS, hub.CDP_EMAIL_TRACKING_URL),new EmailSuppressionService(hub.postgres, emailSuppressionConfigFromEnv()),new RecipientsManagerService(hub.postgres),- null+ null,+ workflowsActivationReporter)mockFetch.mockClear()})@@ -189,7 +193,8 @@ describe('EmailService', () => {new EmailTrackingCodeSigner(hub.ENCRYPTION_SALT_KEYS, hub.CDP_EMAIL_TRACKING_URL),new EmailSuppressionService(hub.postgres, emailSuppressionConfigFromEnv()),new RecipientsManagerService(hub.postgres),- null+ null,+ { report: jest.fn() })expect(serviceWithoutSES.sesV2Client).toBeNull()@@ -286,13 +291,32 @@ describe('EmailService', () => {`"Email integration not found. The sender configured for this step no longer exists — select a new sender in the workflow's email step."`)})- it('should validate if the email domain is not verified', async () => {- invocation.queueParameters = createEmailParams({- from: { integrationId: 2 },- })- const result = await service.executeSendEmail(invocation)- expect(result.error).toMatchInlineSnapshot(`"The selected email integration domain is not verified"`)- })+ it.each([+ ['a workflow send', true, false, 1],+ ['a workflow test send', true, true, 0],+ ['a hog function send', false, false, 0],+ ])(+ 'should fail on an unverified email domain and report it only for a real workflow send: %s',+ async (_name, fromWorkflow, isTest, reports) => {+ invocation.queueParameters = createEmailParams({+ from: { integrationId: 2 },+ })+ const sentInvocation = fromWorkflow+ ? { ...invocation, hogFlow: new FixtureHogFlowBuilder().withTeamId(team.id).build() }+ : invocation+ const result = await service.executeSendEmail(sentInvocation, isTest)+ expect(result.error).toMatchInlineSnapshot(+ `"The selected email integration domain is not verified"`+ )+ expect(workflowsActivationReporter.report.mock.calls).toEqual(+ Array(reports).fill([+ team.id,+ 'workflows send failed',+ { reason: 'unverified_domain', channel: 'email', workflow_id: invocation.functionId },+ ])+ )+ }+ )it('should send identical from and feedback forwarding args', async () => {// This test is important for spam classification - feedback forwarding email MUST match from emailinvocation.queueParameters = createEmailParams({@@ -633,6 +657,7 @@ describe('EmailService', () => {new EmailSuppressionService(hub.postgres, emailSuppressionConfigFromEnv()),new RecipientsManagerService(hub.postgres),null,+ { report: jest.fn() },undefined,{ claimOrReserve } as unknown as RateLimiterService)@@ -805,6 +830,7 @@ describe('EmailService', () => {new EmailSuppressionService(hub.postgres, emailSuppressionConfigFromEnv()),new RecipientsManagerService(hub.postgres),null,+ { report: jest.fn() },undefined,new RateLimiterService(redis, { name: 'workflow-email-backlog-test' }))@@ -876,6 +902,7 @@ describe('EmailService', () => {new EmailSuppressionService(hub.postgres, emailSuppressionConfigFromEnv()),new RecipientsManagerService(hub.postgres),null,+ { report: jest.fn() },undefined,null,{ claimAllOrNothingPair } as unknown as RateLimiterService@@ -1007,6 +1034,7 @@ describe('EmailService', () => {new EmailSuppressionService(hub.postgres, emailSuppressionConfigFromEnv()),new RecipientsManagerService(hub.postgres),null,+ { report: jest.fn() },undefined,null,limiter@@ -1398,6 +1426,33 @@ describe('EmailService', () => {expect(invocation.state.vmState?.stack).toEqual([{ success: false }])})+ it.each([+ ['a workflow send', true, false, 1],+ ['a workflow test send', true, true, 0],+ ['a hog function send', false, false, 0],+ ])(+ 'reports a missing recipient as a blocked send only for a real workflow send: %s',+ async (_name, fromWorkflow, isTest, reports) => {+ invocation.queueParameters = createEmailParams({+ from: { integrationId: 1 },+ to: { email: '', name: 'Test User' },+ })+ const sentInvocation = fromWorkflow+ ? { ...invocation, hogFlow: new FixtureHogFlowBuilder().withTeamId(team.id).build() }+ : invocation++ await service.executeSendEmail(sentInvocation, isTest)++ expect(workflowsActivationReporter.report.mock.calls).toEqual(+ Array(reports).fill([+ team.id,+ 'workflows send blocked',+ { reason: 'missing_recipient', channel: 'email', workflow_id: invocation.functionId },+ ])+ )+ }+ )+describe('SES tenant attribution', () => {it('attributes the send to the team tenant', async () => {sendEmailSpy.mockResolvedValue({ MessageId: 'test-message-id' })diff --git a/nodejs/src/cdp/services/messaging/email.service.ts b/nodejs/src/cdp/services/messaging/email.service.tsindex c67d2f7e592b..7bb4a99cce01 100644--- a/nodejs/src/cdp/services/messaging/email.service.ts+++ b/nodejs/src/cdp/services/messaging/email.service.ts@@ -20,6 +20,7 @@ import { TRANSACTIONAL_EMAIL_EVENT } from '../hogflows/hogflow-sensitivity'import { IntegrationManagerService } from '../managers/integration-manager.service'import { RecipientManagerRecipient, RecipientsManagerService } from '../managers/recipients-manager.service'import { TeamWorkflowsConfigService } from '../managers/team-workflows-config.service'+import { WorkflowsActivationReporter } from '../monitoring/workflows-activation-reporter'import { RateLimiterService } from '../rate-limiter/rate-limiter.service'import { getEmailQueuePriorityClass } from './email-priority'import { selectEmailSenderIntegrationId } from './email-sender-selection'@@ -153,6 +154,16 @@ export class SESThrottleError extends Error {}}+class UnverifiedEmailDomainError extends Error {+ constructor() {+ super('The selected email integration domain is not verified')+ this.name = 'UnverifiedEmailDomainError'+ }+}++const isRealWorkflowSend = (invocation: CyclotronJobInvocationHogFunction, isTest: boolean): boolean =>+ !isTest && 'hogFlow' in invocation+function pickThrottleRetryDelayMs(): number {// Constant 500–1000ms jitter is plenty: the local Valkey bucket already// gates re-dequeue at the configured refill rate, so a quick retry will@@ -424,6 +435,7 @@ export class EmailService {// Required, so that a caller which builds the service without it fails to compile. Pass null// only where no email status is recorded.private emailStatusService: EmailStatusService | null,+ private workflowsActivationReporter: Pick<WorkflowsActivationReporter, 'report'>,private messageAssetsService?: MessageAssetsService,private workflowEmailRateLimiter: RateLimiterService | null = null,private teamEmailRateLimiter: RateLimiterService | null = null@@ -577,6 +589,13 @@ export class EmailService {})}this.emailStatusService?.queueSkippedSend(invocation, 'missing_recipient', isTest)+ if (isRealWorkflowSend(invocation, isTest)) {+ void this.workflowsActivationReporter.report(invocation.teamId, 'workflows send blocked', {+ reason: 'missing_recipient',+ channel: 'email',+ workflow_id: invocation.functionId,+ })+ }result.skipped = trueresult.invocation.state.vmState?.stack.push({ success: false })return result@@ -777,6 +796,13 @@ export class EmailService {result.error = error.messageresult.emailSendResult = { status: 'failed', reason: error.message }result.finished = true+ if (error instanceof UnverifiedEmailDomainError && isRealWorkflowSend(invocation, isTest)) {+ void this.workflowsActivationReporter.report(invocation.teamId, 'workflows send failed', {+ reason: 'unverified_domain',+ channel: 'email',+ workflow_id: invocation.functionId,+ })+ }}}@@ -1059,7 +1085,7 @@ export class EmailService {addLog: ReturnType<typeof createAddLogFunction>): { email: string; name: string } {if (!integration.config.verified) {- throw new Error('The selected email integration domain is not verified')+ throw new UnverifiedEmailDomainError()}if (!integration.config.email || !integration.config.name) {diff --git a/nodejs/src/cdp/services/monitoring/workflows-activation-reporter.test.ts b/nodejs/src/cdp/services/monitoring/workflows-activation-reporter.test.tsnew file mode 100644index 000000000000..a2c7b8f9cc94--- /dev/null+++ b/nodejs/src/cdp/services/monitoring/workflows-activation-reporter.test.ts@@ -0,0 +1,103 @@+import { Team } from '~/types'++import { WorkflowsActivationReporter } from './workflows-activation-reporter'++describe('WorkflowsActivationReporter', () => {+ const team = { id: 2, uuid: 'team-uuid', organization_id: 'org-id' } as Team+ const otherTeam = { id: 3, uuid: 'other-team-uuid', organization_id: 'org-id' } as Team+ let getTeam: jest.Mock+ let capture: jest.Mock+ let reporter: WorkflowsActivationReporter++ beforeEach(() => {+ jest.useFakeTimers({ now: new Date('2026-10-03T10:00:00Z') })+ getTeam = jest.fn((teamId: number) => Promise.resolve([team, otherTeam].find((t) => t.id === teamId) ?? null))+ capture = jest.fn()+ reporter = new WorkflowsActivationReporter({ getTeam }, capture)+ })++ afterEach(() => {+ jest.useRealTimers()+ })++ it('captures each team and reason once per hour', async () => {+ await reporter.report(2, 'workflows send failed', {+ reason: 'unverified_domain',+ channel: 'email',+ workflow_id: 'flow-1',+ })+ await reporter.report(2, 'workflows send failed', {+ reason: 'unverified_domain',+ channel: 'email',+ workflow_id: 'flow-2',+ })+ await reporter.report(3, 'workflows send failed', {+ reason: 'unverified_domain',+ channel: 'email',+ workflow_id: 'flow-9',+ })+ await reporter.report(2, 'workflows send blocked', {+ reason: 'missing_recipient',+ channel: 'email',+ workflow_id: 'flow-1',+ })+ jest.advanceTimersByTime(60 * 60 * 1000)+ await reporter.report(2, 'workflows send failed', {+ reason: 'unverified_domain',+ channel: 'email',+ workflow_id: 'flow-3',+ })++ expect(capture.mock.calls).toEqual([+ [team, 'workflows send failed', { reason: 'unverified_domain', channel: 'email', workflow_id: 'flow-1' }],+ [+ otherTeam,+ 'workflows send failed',+ { reason: 'unverified_domain', channel: 'email', workflow_id: 'flow-9' },+ ],+ [team, 'workflows send blocked', { reason: 'missing_recipient', channel: 'email', workflow_id: 'flow-1' }],+ [team, 'workflows send failed', { reason: 'unverified_domain', channel: 'email', workflow_id: 'flow-3' }],+ ])+ })++ it('skips teams it cannot find', async () => {+ await reporter.report(404, 'workflows send blocked', {+ reason: 'quota_limited',+ channel: 'email',+ workflow_id: 'flow-1',+ })++ expect(capture).not.toHaveBeenCalled()+ })++ it.each([+ ['a failed team lookup', () => getTeam.mockRejectedValueOnce(new Error('postgres is down'))],+ ['a missing team', () => getTeam.mockResolvedValueOnce(null)],+ [+ 'a failed capture',+ () =>+ capture.mockImplementationOnce(() => {+ throw new Error('capture is down')+ }),+ ],+ ])('reports again after %s', async (_name, failOnce) => {+ failOnce()++ await reporter.report(2, 'workflows send blocked', {+ reason: 'quota_limited',+ channel: 'email',+ workflow_id: 'flow-1',+ })+ await reporter.report(2, 'workflows send blocked', {+ reason: 'quota_limited',+ channel: 'email',+ workflow_id: 'flow-2',+ })++ expect(capture).toHaveBeenLastCalledWith(team, 'workflows send blocked', {+ reason: 'quota_limited',+ channel: 'email',+ workflow_id: 'flow-2',+ })+ })+})diff --git a/nodejs/src/cdp/services/monitoring/workflows-activation-reporter.ts b/nodejs/src/cdp/services/monitoring/workflows-activation-reporter.tsnew file mode 100644index 000000000000..c43881115e9f--- /dev/null+++ b/nodejs/src/cdp/services/monitoring/workflows-activation-reporter.ts@@ -0,0 +1,47 @@+import { LRUCache } from 'lru-cache'++import { logger } from '~/common/utils/logger'+import { captureTeamEvent } from '~/common/utils/posthog'+import { Team } from '~/types'++type WorkflowsActivationEvents = {+ 'workflows send blocked': { workflow_id: string; channel: 'email'; reason: 'quota_limited' | 'missing_recipient' }+ 'workflows send failed': { workflow_id: string; channel: 'email'; reason: 'unverified_domain' }+}++const REPORT_INTERVAL_MS = 60 * 60 * 1000++export class WorkflowsActivationReporter {+ private lastReportedAt = new LRUCache<string, number>({ max: 50_000 })++ constructor(+ private teamManager: { getTeam(teamId: number): Promise<Team | null> },+ private capture: typeof captureTeamEvent = captureTeamEvent+ ) {}++ public async report<Event extends keyof WorkflowsActivationEvents>(+ teamId: number,+ event: Event,+ properties: WorkflowsActivationEvents[Event]+ ): Promise<void> {+ const key = `${teamId}:${event}:${properties.reason}`+ const now = Date.now()+ const lastReportedAt = this.lastReportedAt.get(key)+ if (lastReportedAt !== undefined && now - lastReportedAt < REPORT_INTERVAL_MS) {+ return+ }+ this.lastReportedAt.set(key, now)++ try {+ const team = await this.teamManager.getTeam(teamId)+ if (!team) {+ this.lastReportedAt.delete(key)+ return+ }+ this.capture(team, event, properties)+ } catch (error) {+ this.lastReportedAt.delete(key)+ logger.warn('Failed to report a workflows activation event', { teamId, event, error })+ }+ }+}diff --git a/nodejs/src/cdp/templates/test/test-helpers.ts b/nodejs/src/cdp/templates/test/test-helpers.tsindex 6fb026c1b3b1..87e795a81013 100644--- a/nodejs/src/cdp/templates/test/test-helpers.ts+++ b/nodejs/src/cdp/templates/test/test-helpers.ts@@ -221,7 +221,8 @@ export class TemplateTester {new EmailTrackingCodeSigner(config.ENCRYPTION_SALT_KEYS, config.CDP_EMAIL_TRACKING_URL),undefined as any,undefined as any,- null+ null,+ { report: () => Promise.resolve() })return new HogExecutorAsyncService(new HogExecutorService(