:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { DateTime } from 'luxon'
import express from 'ultimate-express'
import { z } from 'zod'
import { ModifiedRequest } from '~/common/api/router'
import type { RedisV2 } from '~/common/redis/redis-v2'
import { logger } from '~/common/utils/logger'
import { UUID, UUIDT, delay } from '~/common/utils/utils'
import { LogRecord } from '~/logs/log-record-avro'
import {
DEFAULT_LOG_TRANSFORMATION_TIMEOUT_MS,
buildLogRecordGlobals,
executeLogTransformation,
resolveLogTransformationInputs,
} from '~/logs/transformations/hog-log-exec'
import { PluginEvent } from '~/plugin-scaffold'
import {
HealthCheckResult,
HealthCheckResultError,
HealthCheckResultOk,
PluginServerService,
PluginsServerConfig,
} from '../types'
import { getAsyncFunctionHandler, getRegisteredAsyncFunctionNames } from './async-function-registry'
import './async-functions'
import { createCdpCoreServices } from './cdp-services'
import { CdpConsumerBaseDeps } from './consumers/cdp-base.consumer'
import {
CdpSourceWebhooksConsumer,
HogFunctionWebhookResult,
SourceWebhookError,
} from './consumers/cdp-source-webhooks.consumer'
import { HogTransformerService, createHogTransformerService } from './hog-transformations/hog-transformer.service'
import { RerunJobManager } from './rerun/rerun-job.manager'
import { RerunRequest } from './rerun/rerun-job.types'
import { HogFlow, HogFlowAction } from './schema/hogflow'
import { BatchExportHogFunctionService, NotFoundError, ParseError } from './services/batch-export-hog-function.service'
import type { CyclotronV2JobProducer } from './services/cyclotron-v2'
import { HogExecutorAsyncService, HogExecutorExecuteAsyncOptions } from './services/hog-executor-async.service'
import { MAX_ASYNC_STEPS } from './services/hog-executor.service'
import { HogInputsService } from './services/hog-inputs.service'
import { counterBatchResolverParked } from './services/hogflows/batch-resolver-resume'
import {
BatchResolverState,
HOGFLOW_BATCH_RESOLVE_QUEUE,
serializeResolverState,
} from './services/hogflows/batch-resolver.types'
import { HogFlowExecutorService, createHogFlowInvocation } from './services/hogflows/hogflow-executor.service'
import { HogFlowManagerService } from './services/hogflows/hogflow-manager.service'
import { checkHogFlowQuotaLimits } from './services/hogflows/hogflow-quota-limiting'
import { TRANSACTIONAL_EMAIL_EVENT } from './services/hogflows/hogflow-sensitivity'
import { matchesWaitUntilCondition } from './services/hogflows/hogflow-utils'
import { IdempotencyKeys } from './services/hogflows/idempotency-keys'
import { WorkflowStepResumeSchema } from './services/hogflows/step-resume.service'
import {
AcceptedSend,
AcceptedSendRequestSchema,
CODE_SEND_TOKEN_CLAIMS,
QueuedSend,
TEMPLATE_SEND_TOKEN_CLAIMS,
TransactionalEmailOutcome,
TransactionalEmailRequest,
TransactionalEmailRequestSchema,
TransactionalEmailStageTimer,
decodeAttachments,
fieldNotAcceptedError,
isRecipientFormatIssue,
queuedSend,
transactionalEmailRecipients,
transactionalEmailRequestsTotal,
} from './services/hogflows/transactional-email'
import { InvocationResultsService } from './services/invocation-results.service'
import { JobQueue } from './services/job-queue/job-queue.interface'
import { GroupsManagerService } from './services/managers/groups-manager.service'
import { HogFunctionManagerService } from './services/managers/hog-function-manager.service'
import { TeamWorkflowsConfigService } from './services/managers/team-workflows-config.service'
import { EmailAttachmentsService } from './services/messaging/email-attachments.service'
import { EMAIL_QUEUE_PRIORITY, EmailQueuePriorityClass, cdpEmailQueuedTotal } from './services/messaging/email-priority'
import { EmailStatusService } from './services/messaging/email-status.service'
import { EmailSuppressionService } from './services/messaging/email-suppression.service'
import { EmailTrackingService } from './services/messaging/email-tracking.service'
import { EmailValidationService } from './services/messaging/email-validation.service'
import { EmailTrackingCodeSigner } from './services/messaging/helpers/tracking-code'
import { RecipientTokensService } from './services/messaging/recipient-tokens.service'
import {
HogWatcherService,
HogWatcherState,
sameWatcherState,
sameWatcherStates,
} from './services/monitoring/hog-watcher.service'
import { NativeDestinationExecutorService } from './services/native-destination-executor.service'
import { SegmentDestinationExecutorService } from './services/segment-destination-executor.service'
import { HOG_FUNCTION_TEMPLATES } from './templates'
import { HogFunctionInvocationGlobals, HogFunctionType, MinimalLogEntry } from './types'
import {
convertToHogFunctionInvocationGlobals,
isConvertibleClickHouseEvent,
isNativeHogFunction,
isSegmentPluginHogFunction,
sanitizeLogMessage,
} from './utils'
import { dualRead, dualWrite } from './utils/dual-store'
import { convertToHogFunctionFilterGlobal } from './utils/hog-function-filtering'
import { buildHogFunctionInvocations } from './utils/invocation-utils'
import { PosthogJwtAudience } from './utils/jwt-utils'
import { ScopedServiceJwt } from './utils/scoped-service-jwt'
import { parseWorkflowStepDispatchKey } from './utils/workflow-step-dispatch-key'
// Allowlist of safe content types for webhook responses to prevent XSS
const SAFE_CONTENT_TYPES = new Set([
'text/plain',
'text/csv',
'application/json',
'application/octet-stream',
'application/xml',
'image/gif',
'image/png',
'image/jpeg',
'image/webp',
])
function sanitizeContentType(contentType: string | undefined, fallback: string): string {
if (!contentType) {
return fallback
}
const normalized = contentType.toLowerCase().trim().split(';')[0].trim()
if (SAFE_CONTENT_TYPES.has(normalized)) {
return normalized
}
return fallback
}
// Matches the default email template's `to.email` value: `{{ person.properties.email }}`, whitespace-tolerant.
// Anything else (custom property, computed Liquid, static address) makes the dedupe key diverge from the
// actual send target — see `canDedupeByEmail`.
const DEFAULT_EMAIL_TO_TEMPLATE_RE = /^\s*\{\{\s*person\.properties\.email\s*\}\}\s*$/
// One account assignment filter is spread over these three keys.
const ASSIGNMENT_FILTER_KEYS = ['assignment_status', 'assigned_to_user_ids', 'all_roles_unassigned'] as const
function canDedupeByEmail(hogFlow: { actions?: unknown }): boolean {
if (!Array.isArray(hogFlow.actions)) {
return false
}
const emailActions = hogFlow.actions.filter((action: any) => action?.type === 'function_email')
if (emailActions.length === 0) {
return false
}
return emailActions.every((action: any) => {
const toEmail = action?.config?.inputs?.email?.value?.to?.email
return typeof toEmail === 'string' && DEFAULT_EMAIL_TO_TEMPLATE_RE.test(toEmail)
})
}
export type CdpApiConfig = PluginsServerConfig
export type CdpApiDeps = CdpConsumerBaseDeps & {
/** Holds caller-supplied idempotency keys. Null or unset makes the store report itself unavailable. */
idempotencyValkey?: RedisV2 | null
}
// Null means the cohort asks to wait but its deadline is unreadable. Queuing it unparked would send to a
// cohort that is still calculating, so the caller rejects it.
function parseRealtimeCohort(value: unknown): BatchResolverState['realtimeCohort'] | null {
if (typeof value !== 'object' || value === null) {
return undefined
}
const { id, team_id, wait_until } = value as { id?: unknown; team_id?: unknown; wait_until?: unknown }
if (!Number.isInteger(id) || !Number.isInteger(team_id)) {
return undefined
}
if (wait_until === undefined || wait_until === null) {
return { id: id as number, teamId: team_id as number }
}
const waitUntil = typeof wait_until === 'string' ? new Date(wait_until) : null
if (!waitUntil || isNaN(waitUntil.getTime())) {
return null
}
return { id: id as number, teamId: team_id as number, waitUntil: waitUntil.toISOString() }
}
const ACCEPTED_SEND_FINGERPRINT_MARKER = 'accepted_send'
const transactionalEmailFingerprint = (
hogFlowId: string,
{ to, distinct_id, variables }: TransactionalEmailRequest,
send: QueuedSend | undefined
): string => {
const templateRequest = [hogFlowId, to, distinct_id, variables]
return IdempotencyKeys.fingerprint(
send ? [...templateRequest, ACCEPTED_SEND_FINGERPRINT_MARKER, withSenderTheCallerChose(send)] : templateRequest
)
}
type CallerChosenSender = Partial<NonNullable<AcceptedSend['from']>>
const withSenderTheCallerChose = ({
from,
...send
}: QueuedSend): Omit<QueuedSend, 'from'> & { from?: CallerChosenSender } => ({
...send,
from: senderTheCallerChose(from),
})
const senderTheCallerChose = (from: AcceptedSend['from']): CallerChosenSender | undefined => {
if (!from) {
return undefined
}
return from.email ? { email: from.email, name: from.name } : { integrationId: from.integrationId, name: from.name }
}
export class CdpApi {
private hogExecutorAsync: HogExecutorAsyncService
private hogInputsService: HogInputsService
private nativeDestinationExecutorService: NativeDestinationExecutorService
private segmentDestinationExecutorService: SegmentDestinationExecutorService
private hogFunctionManager: HogFunctionManagerService
private hogFlowManager: HogFlowManagerService
private hogFlowExecutor: HogFlowExecutorService
private hogWatcher: HogWatcherService
private hogWatcherMirror: HogWatcherService
private hogTransformer: HogTransformerService
private invocationResultsService: InvocationResultsService
private rerunJobManager: RerunJobManager | null = null
private cdpSourceWebhooksConsumer: CdpSourceWebhooksConsumer
private hogQueue: JobQueue
private hogflowQueue: JobQueue
private emailTrackingService: EmailTrackingService
private recipientTokensService: RecipientTokensService
private batchExportHogFunctionService: BatchExportHogFunctionService
private groupsManager: GroupsManagerService
private batchResolverProducer: CyclotronV2JobProducer | null
// Scoped auth for the reschedule_parked and cancel routes (exempted from the shared
// internal-secret middleware): Django mints per-call JWTs pinned to a team + workflow,
// one audience per route. Disabled when the key isn't provisioned — the routes then fail closed.
private rescheduleJwt: ScopedServiceJwt
private cancelInvocationsJwt: ScopedServiceJwt
private cancelBatchJwt: ScopedServiceJwt
private stepResumeJwt: ScopedServiceJwt
private transactionalEmailJwt: ScopedServiceJwt
private transactionalEmailIdempotencyKeys: IdempotencyKeys
private emailSuppressionService: EmailSuppressionService
private emailAttachmentsService: EmailAttachmentsService
private emailValidationService: EmailValidationService
private teamWorkflowsConfigService: TeamWorkflowsConfigService
private emailStatusService: EmailStatusService
private resumeBatchJwt: ScopedServiceJwt
constructor(
private config: PluginsServerConfig,
private deps: CdpApiDeps,
jobQueues: { hogQueue: JobQueue; hogflowQueue: JobQueue },
batchResolverProducer: CyclotronV2JobProducer | null = null
) {
const services = createCdpCoreServices(config, deps, 'cdp-api-redis')
this.hogFunctionManager = services.hogFunctionManager
this.hogFlowManager = services.hogFlowManager
this.recipientTokensService = services.recipientTokensService
this.emailSuppressionService = services.emailSuppressionService
this.emailAttachmentsService = services.emailAttachmentsService
this.emailValidationService = services.emailValidationService
this.teamWorkflowsConfigService = services.teamWorkflowsConfigService
this.hogExecutorAsync = services.hogExecutorAsync
this.hogInputsService = services.hogInputsService
this.hogFlowExecutor = services.hogFlowExecutor
this.nativeDestinationExecutorService = services.nativeDestinationExecutorService
this.segmentDestinationExecutorService = services.segmentDestinationExecutorService
this.hogWatcher = services.hogWatcher
this.hogWatcherMirror = services.hogWatcherMirror
this.invocationResultsService = services.invocationResultsService
this.emailStatusService = services.emailStatusService
// API-only services. The hog-transformer's monitoring service reuses the same
// resolved outputs registry as the core CDP services — no separate construction.
this.hogTransformer = createHogTransformerService(config, {
...deps,
monitoringOutputs: services.outputs,
})
this.hogQueue = jobQueues.hogQueue
this.hogflowQueue = jobQueues.hogflowQueue
this.cdpSourceWebhooksConsumer = new CdpSourceWebhooksConsumer(
config,
deps,
jobQueues,
new IdempotencyKeys(deps.idempotencyValkey ?? null, 'workflow-webhook')
)
this.emailTrackingService = new EmailTrackingService(
this.hogFunctionManager,
this.hogFlowManager,
services.hogFunctionMonitoringService,
services.capturedEventsService,
services.teamWorkflowsConfigService,
new EmailTrackingCodeSigner(config.ENCRYPTION_SALT_KEYS, config.CDP_EMAIL_TRACKING_URL),
services.emailSuppressionService,
services.emailStatusService,
services.redis
)
this.groupsManager = new GroupsManagerService(deps.teamManager, deps.groupRepository)
this.batchExportHogFunctionService = new BatchExportHogFunctionService(
config.SITE_URL,
deps.teamManager,
this.groupsManager,
this.hogFunctionManager,
this.hogExecutorAsync,
this.hogWatcher,
this.invocationResultsService,
this.hogWatcherMirror
)
this.batchResolverProducer = batchResolverProducer
this.rescheduleJwt = new ScopedServiceJwt(
PosthogJwtAudience.WORKFLOWS_RESCHEDULE_PARKED,
config.WORKFLOWS_RESCHEDULE_JWT_SECRET || ''
)
this.cancelInvocationsJwt = new ScopedServiceJwt(
PosthogJwtAudience.WORKFLOWS_CANCEL_INVOCATIONS,
config.WORKFLOWS_CANCEL_JWT_SECRET || ''
)
this.cancelBatchJwt = new ScopedServiceJwt(
PosthogJwtAudience.WORKFLOWS_CANCEL_BATCH,
config.WORKFLOWS_CANCEL_JWT_SECRET || ''
)
this.stepResumeJwt = new ScopedServiceJwt(
PosthogJwtAudience.WORKFLOWS_STEP_RESUME,
config.WORKFLOWS_STEP_RESUME_JWT_SECRET || ''
)
this.transactionalEmailJwt = new ScopedServiceJwt(
PosthogJwtAudience.WORKFLOWS_SEND_TRANSACTIONAL_EMAIL,
config.WORKFLOWS_TRANSACTIONAL_EMAIL_JWT_SECRET || ''
)
this.transactionalEmailIdempotencyKeys = new IdempotencyKeys(
deps.idempotencyValkey ?? null,
'transactional-email'
)
// The same workers that wake parked steps wake parked batch resolvers, so they share the
// key. The audience keeps a step-resume token from waking a batch, and the reverse.
this.resumeBatchJwt = new ScopedServiceJwt(
PosthogJwtAudience.WORKFLOWS_RESUME_BATCH,
config.WORKFLOWS_STEP_RESUME_JWT_SECRET || ''
)
}
public get service(): PluginServerService {
return {
id: 'cdp-api',
onShutdown: async () => await this.stop(),
healthcheck: () => this.isHealthy() ?? new HealthCheckResultError('CDP API is not healthy', {}),
}
}
async start(): Promise<void> {
// CdpSourceWebhooksConsumer.start() calls startAsProducer on both queues
await this.cdpSourceWebhooksConsumer.start()
// Rerun endpoints don't run the work — they just enqueue a wrapper
// job onto the cyclotron-v2 'rerun' queue. A dedicated consumer
// (`CdpRerunWorkerConsumer`) deployed as PLUGIN_SERVER_MODE=cdp-rerun-worker
// pages ClickHouse, rehydrates invocations, and commits progress back
// to the wrapper job via reschedule(state).
if (this.config.CYCLOTRON_NODE_DATABASE_URL) {
this.rerunJobManager = new RerunJobManager({
dbUrl: this.config.CYCLOTRON_NODE_DATABASE_URL,
maxCount: this.config.HOG_INVOCATION_RERUN_MAX_COUNT,
runStartPagingTeams: this.config.CDP_RERUN_RUN_START_PAGING_TEAMS,
})
await this.rerunJobManager.connect()
}
}
async stop(): Promise<void> {
// CdpSourceWebhooksConsumer.stop() calls stopProducer on both queues
await Promise.all([
this.cdpSourceWebhooksConsumer.stop(),
this.batchExportHogFunctionService.stop(),
this.rerunJobManager?.disconnect() ?? Promise.resolve(),
this.invocationResultsService.stop(),
this.emailAttachmentsService.stop(),
])
}
isHealthy(): HealthCheckResult {
// NOTE: There isn't really anything to check for here so we are just always healthy
return new HealthCheckResultOk()
}
router(): express.Router {
const router = express.Router()
const asyncHandler =
(fn: (req: ModifiedRequest, res: express.Response) => Promise<void>) =>
(req: ModifiedRequest, res: express.Response, next: express.NextFunction): Promise<void> =>
fn(req, res).catch(next)
// API routes (authentication handled globally by middleware)
router.post('/api/projects/:team_id/hog_functions/:id/invocations', asyncHandler(this.postFunctionInvocation))
router.post('/api/projects/:team_id/hog_flows/:id/invocations', asyncHandler(this.postHogflowInvocation))
router.post(
'/api/projects/:team_id/hog_flows/:id/scheduled_invocations',
asyncHandler(this.postHogflowScheduledInvocation)
)
router.post(
'/api/projects/:team_id/hog_flows/:id/batch_invocations/:parent_run_id',
asyncHandler(this.postHogFlowBatchInvocation)
)
router.post(
'/api/projects/:team_id/hog_functions/:id/rerun',
asyncHandler(this.postRerunInvocations('hog_function'))
)
router.post('/api/projects/:team_id/hog_flows/:id/rerun', asyncHandler(this.postRerunInvocations('hog_flow')))
router.get('/api/projects/:team_id/hog_flows/:id/in_flight_count', asyncHandler(this.getHogFlowInFlightCount))
router.post(
'/api/projects/:team_id/hog_flows/:id/reschedule_parked',
asyncHandler(this.postHogFlowRescheduleParked)
)
router.post(
'/api/projects/:team_id/hog_flows/:id/invocations/cancel',
asyncHandler(this.postHogFlowCancelInvocations)
)
router.post(
'/api/projects/:team_id/hog_flows/:id/batch_jobs/:batch_job_id/cancel',
asyncHandler(this.postHogFlowCancelBatchJob)
)
router.post(
'/api/projects/:team_id/hog_flows/:id/batch_jobs/:batch_job_id/resume',
asyncHandler(this.postHogFlowResumeBatchJob)
)
router.post('/api/projects/:team_id/workflow_steps/resume', asyncHandler(this.postWorkflowStepResume))
router.post(
'/api/projects/:team_id/hog_flows/:id/transactional_email',
asyncHandler(this.postTransactionalEmail(TransactionalEmailRequestSchema, TEMPLATE_SEND_TOKEN_CLAIMS))
)
router.post(
'/api/projects/:team_id/hog_flows/:id/transactional_email/send',
asyncHandler(this.postTransactionalEmail(AcceptedSendRequestSchema, CODE_SEND_TOKEN_CLAIMS))
)
router.get('/api/projects/:team_id/hog_functions/:id/status', asyncHandler(this.getFunctionStatus()))
router.patch('/api/projects/:team_id/hog_functions/:id/status', asyncHandler(this.patchFunctionStatus()))
router.get('/api/hog_functions/states', asyncHandler(this.getFunctionStates()))
router.get('/api/hog_function_templates', this.getHogFunctionTemplates)
router.post('/api/messaging/generate_preferences_token', asyncHandler(this.generatePreferencesToken()))
router.get('/api/messaging/validate_preferences_token/:token', asyncHandler(this.validatePreferencesToken()))
router.post(
'/api/projects/:team_id/hog_functions/:hog_function_id/batch_export_invocations',
asyncHandler(this.handleBatchExportHogFunction())
)
const publicBodySizeLimit = (req: ModifiedRequest, res: express.Response, next: express.NextFunction): void => {
if (req.rawBody && req.rawBody.length > 512_000) {
res.status(413).json({ error: 'Request entity too large' })
return
}
next()
}
// Public routes (excluded from authentication by middleware)
router.post(
'/public/webhooks/dwh/:webhook_id',
publicBodySizeLimit,
asyncHandler(this.handleWarehouseSourceWebhook())
)
router.post('/public/webhooks/:webhook_id', publicBodySizeLimit, asyncHandler(this.handleWebhook()))
router.get('/public/webhooks/:webhook_id', asyncHandler(this.handleWebhook()))
router.get('/public/m/pixel', asyncHandler(this.getEmailTrackingPixel()))
router.post('/public/m/ses_webhook', publicBodySizeLimit, express.text(), asyncHandler(this.postSesWebhook()))
router.get('/public/m/redirect', asyncHandler(this.getEmailTrackingRedirect()))
return router
}
private getHogFunctionTemplates = (req: ModifiedRequest, res: express.Response): void => {
res.json(HOG_FUNCTION_TEMPLATES)
}
private getFunctionStatus =
() =>
async (req: ModifiedRequest, res: express.Response): Promise<void> => {
const { id } = req.params
const summary = await dualRead(
'hog-watcher.getPersistedState',
() => this.hogWatcher.getPersistedState(id),
() => this.hogWatcherMirror.getPersistedState(id),
sameWatcherState
)
res.json(summary)
}
private patchFunctionStatus =
() =>
async (req: ModifiedRequest, res: express.Response): Promise<void> => {
const { id } = req.params
const { state } = req.body
// Check that state is valid
if (!Object.values(HogWatcherState).includes(state)) {
res.status(400).json({ error: 'Invalid state' })
return
}
const summary = await dualRead(
'hog-watcher.getPersistedState',
() => this.hogWatcher.getPersistedState(id),
() => this.hogWatcherMirror.getPersistedState(id),
sameWatcherState
)
const hogFunction = await this.hogFunctionManager.fetchHogFunction(id)
if (!hogFunction) {
res.status(404).json({ error: 'Hog function not found' })
return
}
// Only allow patching the status if it is different from the current status
if (summary.state !== state) {
await dualWrite(
'hog-watcher.forceStateChange',
() => this.hogWatcher.forceStateChange(hogFunction, state),
() => this.hogWatcherMirror.forceStateChange(hogFunction, state)
)
}
// Hacky - wait for a little to give a chance for the state to change
await delay(100)
res.json(
await dualRead(
'hog-watcher.getPersistedState',
() => this.hogWatcher.getPersistedState(id),
() => this.hogWatcherMirror.getPersistedState(id),
sameWatcherState
)
)
}
private getFunctionStates =
() =>
async (req: ModifiedRequest, res: express.Response): Promise<void> => {
try {
const allStates = await dualRead(
'hog-watcher.getAllFunctionStates',
() => this.hogWatcher.getAllFunctionStates(),
() => this.hogWatcherMirror.getAllFunctionStates(),
sameWatcherStates
)
// Transform the data for better consumption by Grafana and sort by tokens ascending
const statesArray = Object.entries(allStates)
.map(([functionId, state]) => ({
function_id: functionId,
state: HogWatcherState[state.state], // Convert numeric state to readable string
tokens: state.tokens,
state_numeric: state.state,
}))
.sort((a, b) => b.state_numeric - a.state_numeric)
const hogFunctions = await this.hogFunctionManager.getHogFunctions(
statesArray.map((x) => x.function_id)
)
const results = statesArray.map((x) => ({
...x,
function_name: hogFunctions[x.function_id]?.name,
function_team_id: hogFunctions[x.function_id]?.team_id,
function_type: hogFunctions[x.function_id]?.type,
function_enabled: hogFunctions[x.function_id]?.enabled && !hogFunctions[x.function_id]?.deleted,
}))
res.json({
results,
total: results.length,
})
} catch (error) {
logger.error('[CdpApi] Error getting all function states', error)
res.status(500).json({ error: 'Failed to get function states' })
}
}
private postFunctionInvocation = async (req: ModifiedRequest, res: express.Response): Promise<any> => {
try {
const { id, team_id } = req.params
const { clickhouse_event, mock_async_functions, configuration, invocation_id } = req.body
let { globals } = req.body
// Redact configuration: it carries function inputs (auth headers, API keys) that must not land in logs
logger.info('⚡️', 'Received invocation', {
id,
team_id,
body: { ...req.body, configuration: configuration ? '[redacted]' : undefined },
})
const invocationID = invocation_id ?? new UUIDT().toString()
// Check the invocationId is a valid UUID
if (!UUID.validateString(invocationID)) {
res.status(400).json({ error: 'Invalid invocation ID' })
return
}
const isNewFunction = req.params.id === 'new'
const hogFunction = isNewFunction
? null
: await this.hogFunctionManager.fetchHogFunction(req.params.id).catch(() => null)
const team = await this.deps.teamManager.getTeam(parseInt(team_id)).catch(() => null)
if (!team) {
return res.status(404).json({ error: 'Team not found' })
}
globals = isConvertibleClickHouseEvent(clickhouse_event)
? convertToHogFunctionInvocationGlobals(clickhouse_event, team, this.config.SITE_URL)
: globals
const functionType: string | undefined = configuration?.type ?? hogFunction?.type
if (functionType === 'transformation_log') {
// Log transformations run against a log record, not an event
if (!globals?.record || typeof globals.record !== 'object' || Array.isArray(globals.record)) {
res.status(400).json({ error: 'Missing record' })
return
}
} else if (!globals || !globals.event) {
res.status(400).json({ error: 'Missing event' })
return
}
// NOTE: We allow the hog function to be null if it is a "new" hog function
// The real security happens at the django layer so this is more of a sanity check
if (!isNewFunction && (!hogFunction || hogFunction.team_id !== team.id)) {
return res.status(404).json({ error: 'Hog function not found' })
}
// We use the provided config if given, otherwise the function's config
const compoundConfiguration: HogFunctionType = {
...hogFunction,
...configuration,
team_id: team.id,
}
let logs: MinimalLogEntry[] = []
let result: any = null
const errors: any[] = []
const triggerGlobals: HogFunctionInvocationGlobals = {
...globals,
// Sample globals from the app can carry a placeholder project, and a test send with
// mock_async_functions off delivers for real, so the server's project always wins.
project: {
...globals.project,
id: team.id,
name: team.name,
url: `${this.config.SITE_URL}/project/${team.id}`,
},
}
if (['destination', 'internal_destination'].includes(compoundConfiguration.type)) {
const {
invocations,
logs: filterLogs,
metrics: filterMetrics,
} = await buildHogFunctionInvocations(this.hogInputsService, [compoundConfiguration], triggerGlobals)
// Add metrics to the logs
filterMetrics.forEach((metric) => {
if (metric.metric_name === 'filtered') {
logs.push({
level: 'info',
timestamp: DateTime.now(),
message: `Mapping trigger not matching filters was ignored.`,
})
}
})
filterLogs.forEach((log) => {
logs.push(log)
})
for (const invocation of invocations) {
invocation.id = invocationID
const sensitiveValues = this.hogExecutorAsync.hogExecutor.getSensitiveValues(
invocation.hogFunction,
invocation.state.globals.inputs ?? {}
)
const options: HogExecutorExecuteAsyncOptions = buildHogExecutorAsyncOptions(
mock_async_functions,
logs,
sensitiveValues
)
options.isTest = true
let response: any = null
if (isNativeHogFunction(compoundConfiguration)) {
response = await this.nativeDestinationExecutorService.execute(invocation)
} else if (isSegmentPluginHogFunction(compoundConfiguration)) {
response = await this.segmentDestinationExecutorService.execute(invocation)
} else {
response = await this.hogExecutorAsync.executeWithAsyncFunctions(invocation, options)
}
logs = logs.concat(response.logs)
if (response.error) {
errors.push(response.error)
}
}
const wasSkipped = invocations.length === 0
res.json({
result: result,
status: errors.length > 0 ? 'error' : wasSkipped ? 'skipped' : 'success',
errors: errors.map((e) => String(e)),
logs: logs,
})
} else if (compoundConfiguration.type === 'transformation') {
// NOTE: We override the ID so that the transformer doesn't cache the result
// TODO: We could do this with a "special" ID to indicate no caching...
compoundConfiguration.id = new UUIDT().toString()
const pluginEvent: PluginEvent = {
...triggerGlobals.event,
ip:
typeof triggerGlobals.event.properties.$ip === 'string'
? triggerGlobals.event.properties.$ip
: null,
site_url: triggerGlobals.project.url,
team_id: triggerGlobals.project.id,
now: '',
}
const response = await this.hogTransformer.transformEvent(pluginEvent, [compoundConfiguration])
result = response.event
for (const invocationResult of response.invocationResults) {
logs = logs.concat(invocationResult.logs)
if (invocationResult.error) {
errors.push(invocationResult.error)
}
}
const wasSkipped = response.invocationResults.some((r) =>
r.metrics.some((m) => m.metric_name === 'filtered')
)
res.json({
result: result,
status: errors.length > 0 ? 'error' : wasSkipped ? 'skipped' : 'success',
errors: errors.map((e) => String(e)),
logs: logs,
})
} else if (compoundConfiguration.type === 'transformation_log') {
const mock = globals.record as Record<string, unknown>
const toStringMap = (value: unknown): Record<string, string> => {
const map: Record<string, string> = {}
if (value && typeof value === 'object' && !Array.isArray(value)) {
for (const [key, entry] of Object.entries(value as Record<string, unknown>)) {
if (entry !== null && entry !== undefined) {
map[key] = typeof entry === 'string' ? entry : JSON.stringify(entry)
}
}
}
return map
}
// Mock log record from the request; trace/span ids stay null (they are
// read-only in transformations and not meaningful for a test run).
const record: LogRecord = {
uuid: invocationID,
trace_id: null,
span_id: null,
trace_flags: null,
timestamp: typeof mock.timestamp === 'number' ? mock.timestamp : DateTime.now().toMillis() * 1e6,
observed_timestamp: typeof mock.observed_timestamp === 'number' ? mock.observed_timestamp : null,
body: typeof mock.body === 'string' ? mock.body : null,
severity_text: typeof mock.severity_text === 'string' ? mock.severity_text : null,
severity_number: typeof mock.severity_number === 'number' ? mock.severity_number : null,
service_name: typeof mock.service_name === 'string' ? mock.service_name : null,
resource_attributes: toStringMap(mock.resource_attributes),
instrumentation_scope:
typeof mock.instrumentation_scope === 'string' ? mock.instrumentation_scope : null,
event_name: typeof mock.event_name === 'string' ? mock.event_name : null,
attributes: toStringMap(mock.attributes),
bytes_uncompressed: null,
}
const hogGlobals = buildLogRecordGlobals(record, triggerGlobals.project, {})
try {
hogGlobals.inputs = resolveLogTransformationInputs(
compoundConfiguration,
hogGlobals,
DEFAULT_LOG_TRANSFORMATION_TIMEOUT_MS
).inputs
} catch (e) {
return res.json({
result: null,
status: 'error',
errors: [String(e)],
logs,
})
}
// Derive from the resolved inputs (which merge inputs + encrypted_inputs) like the
// destination test path does — Django resolves stored secrets into `inputs`, so
// collecting from `encrypted_inputs` alone would leave them unredacted in test logs.
const sensitiveValues = this.hogExecutorAsync.hogExecutor.getSensitiveValues(
compoundConfiguration,
(hogGlobals.inputs ?? {}) as Record<string, any>
)
const outcome = executeLogTransformation(compoundConfiguration.bytecode, record, hogGlobals, {
sensitiveValues,
})
logs = logs.concat(
outcome.logs.map((message) => ({
level: 'info' as const,
timestamp: DateTime.now(),
message,
}))
)
if (outcome.status === 'failed') {
errors.push(outcome.error)
} else if (outcome.status === 'dropped') {
logs.push({
level: 'info',
timestamp: DateTime.now(),
message: 'Record dropped by transformation.',
})
}
// Same record shape the function saw (hex ids, string maps) — null when dropped
result =
outcome.status === 'dropped'
? null
: buildLogRecordGlobals(record, triggerGlobals.project, {}).record
res.json({
result: result,
status: errors.length > 0 ? 'error' : 'success',
errors: errors.map((e) => String(e)),
logs: logs,
})
} else {
return res.status(400).json({ error: 'Invalid function type' })
}
} catch (e) {
console.error(e)
res.status(500).json({ errors: [e.message] })
} finally {
await this.invocationResultsService.flush()
}
}
private postHogflowInvocation = async (req: ModifiedRequest, res: express.Response): Promise<any> => {
try {
const { id, team_id } = req.params
const { clickhouse_event, configuration, invocation_id, current_action_id, mock_async_functions } = req.body
// Redact configuration: it carries action inputs (auth headers, API keys) that must not land in logs
logger.info('⚡️', 'Received hogflow invocation', {
id,
team_id,
body: { ...req.body, configuration: configuration ? '[redacted]' : undefined },
})
const invocationID = invocation_id ?? new UUIDT().toString()
// Check the invocationId is a valid UUID
if (!UUID.validateString(invocationID)) {
res.status(400).json({ error: 'Invalid invocation ID' })
return
}
const isNewHogFlow = req.params.id === 'new'
const hogFlow = isNewHogFlow ? null : await this.hogFlowManager.getHogFlow(req.params.id)
const team = await this.deps.teamManager.getTeam(parseInt(team_id)).catch(() => null)
if (!team) {
return res.status(404).json({ error: 'Team not found' })
}
// NOTE: We allow the hog flow to be null if it is a "new" hog flow
// The real security happens at the django layer so this is more of a sanity check
if (!isNewHogFlow && (!hogFlow || hogFlow.team_id !== team.id)) {
return res.status(404).json({ error: 'Hog flow not found' })
}
const globals: HogFunctionInvocationGlobals | null = isConvertibleClickHouseEvent(clickhouse_event)
? convertToHogFunctionInvocationGlobals(
clickhouse_event,
team,
this.config.SITE_URL ?? 'http://localhost:8000'
)
: req.body.globals
if (!globals || !globals.event) {
return res.status(400).json({ error: 'Missing event' })
}
// We use the provided config if given, otherwise the flow's config
const compoundConfiguration = {
...hogFlow,
...configuration,
team_id: team.id,
}
// Mirror real execution: resolve groups server-side from the event's $groups so test-run
// conditionals branch on group properties. Only resolve when the caller didn't supply
// groups, so hand-edited test payloads are respected.
if (!globals.groups || Object.keys(globals.groups).length === 0) {
globals.groups = await this.groupsManager.getGroupsForEvent(
team.id,
globals.event.properties,
`${this.config.SITE_URL ?? 'http://localhost:8000'}/project/${team.id}`
)
}
const triggerGlobals: HogFunctionInvocationGlobals = {
...globals,
project: {
id: team.id,
name: team.name,
url: `${this.config.SITE_URL ?? 'http://localhost:8000'}/project/${team.id}`,
},
}
const filterGlobals = convertToHogFunctionFilterGlobal({
event: globals.event,
person: globals.person,
groups: globals.groups,
variables: globals.variables || {},
})
const invocation = createHogFlowInvocation(triggerGlobals, compoundConfiguration, filterGlobals)
// Real event ingestion evaluates trigger filters before creating an invocation. A test run has to
// execute the trigger action itself so callers can verify whether their supplied globals match.
// Without this explicit position, executeCurrentAction starts after the trigger by design.
const startingActionId =
current_action_id ??
compoundConfiguration.actions?.find((action: HogFlowAction) => action.type === 'trigger')?.id
invocation.state.currentAction = startingActionId
? {
id: startingActionId,
startedAtTimestamp: Date.now(),
}
: undefined
const logs: MinimalLogEntry[] = []
// In production a wait_until_condition step's "events to wait for" are evaluated by the
// subscription matcher against incoming events (never by the executor), so a plain
// executeCurrentAction could not advance past one. Simulate the matcher here: when the
// supplied test event matches, tag the invocation the same way a real match would, and
// the handler advances to the next step.
const currentAction: HogFlowAction | undefined = current_action_id
? compoundConfiguration.actions?.find((a: HogFlowAction) => a.id === current_action_id)
: undefined
if (currentAction?.type === 'wait_until_condition' && invocation.state.currentAction) {
const matched = await matchesWaitUntilCondition(currentAction, filterGlobals, {
hogFlowId: isNewHogFlow ? 'new' : id,
actionId: currentAction.id,
})
if (matched) {
invocation.state.currentAction.eventMatched = true
invocation.state.currentAction.eventMatchedEvent = globals.event.event
invocation.state.currentAction.eventMatchedEventUuid = globals.event.uuid
invocation.state.currentAction.eventMatchedEventTimestamp = globals.event.timestamp
}
logs.push({
level: 'info',
timestamp: DateTime.now(),
message: matched
? `Test event '${globals.event.event}' matched the wait conditions`
: `Test event '${globals.event.event}' did not match the wait conditions - the workflow would continue waiting`,
})
}
// Redact the flow's decrypted secret inputs from the mocked async-function logs, so a test
// run can't echo a stored credential (e.g. an Authorization header) back to the caller.
const sensitiveValues = await this.hogFlowExecutor.getSensitiveValues(compoundConfiguration)
const options: HogExecutorExecuteAsyncOptions = buildHogExecutorAsyncOptions(
mock_async_functions,
logs,
sensitiveValues
)
options.isTest = true
const result = await this.hogFlowExecutor.executeCurrentAction(invocation, { hogExecutorOptions: options })
res.json({
nextActionId: result.skipped ? null : result.invocation.state.currentAction?.id,
status: result.error ? 'error' : result.skipped ? 'skipped' : 'success',
errors: result.error ? [result.error] : [],
logs: [...result.logs, ...logs],
variables: result.invocation.state.variables ?? {},
execResult: result.execResult ?? null,
...(result.emailSendResult ? { emailSendResult: result.emailSendResult } : {}),
})
} catch (e) {
console.error(e)
res.status(500).json({ error: [e.message] })
}
}
private postHogflowScheduledInvocation = async (req: ModifiedRequest, res: express.Response): Promise<any> => {
try {
const { id, team_id } = req.params
const { variables } = req.body
logger.info('⚡️', 'Received hogflow scheduled invocation', { id, team_id })
const team = await this.deps.teamManager.getTeam(parseInt(team_id)).catch(() => null)
if (!team) {
return res.status(404).json({ error: 'Team not found' })
}
const hogFlow = await this.hogFlowManager.getHogFlow(id)
if (!hogFlow || hogFlow.team_id !== team.id) {
return res.status(404).json({ error: 'Workflow not found' })
}
if (hogFlow.trigger?.type !== 'schedule') {
return res.status(400).json({ error: 'Workflow trigger must be of type "schedule"' })
}
// Build a synthetic event for the scheduled run. Schedule triggers don't have a real
// event, but the executor expects one to populate globals.event used by downstream actions.
const syntheticEvent: HogFunctionInvocationGlobals['event'] = {
uuid: new UUIDT().toString(),
event: '$workflow_scheduled',
distinct_id: `workflow-${hogFlow.id}`,
timestamp: DateTime.now().toISO(),
url: '',
properties: {},
elements_chain: '',
}
const triggerGlobals: HogFunctionInvocationGlobals = {
event: syntheticEvent,
project: {
id: team.id,
name: team.name,
url: `${this.config.SITE_URL ?? 'http://localhost:8000'}/project/${team.id}`,
},
variables: variables ?? {},
}
const filterGlobals = convertToHogFunctionFilterGlobal({
event: syntheticEvent,
person: undefined,
groups: {},
variables: variables ?? {},
})
const invocation = createHogFlowInvocation(triggerGlobals, hogFlow, filterGlobals)
// Queued before queueInvocations serializes the invocation, so the
// `state.firstScheduledAt` stamp reaches cyclotron.
this.invocationResultsService.invocationResultsRowsService.queueLifecycleRow(invocation, 'running')
try {
await this.hogflowQueue.queueInvocations([invocation])
} catch (error) {
this.invocationResultsService.invocationResultsRowsService.dropQueuedRowsFor([invocation.id])
throw error
}
// Only the lifecycle sink, which swallows its own produce failures. Flushing every sink
// would fail an enqueued run on an unrelated sink's error, and the caller would retry it.
await this.invocationResultsService.invocationResultsRowsService.flush()
res.json({ status: 'queued', invocation_id: invocation.id })
} catch (e) {
logger.error('Error handling hogflow scheduled invocation', { error: e })
res.status(500).json({ error: [e.message] })
}
}
// Rerun endpoints don't run the work — they just enqueue a wrapper job
// onto the cyclotron-v2 'rerun' queue. The dedicated `CdpRerunWorkerConsumer`
// picks it up, pages ClickHouse, rehydrates invocations onto the regular
// queue, and commits progress back to the wrapper job's state.
private postRerunInvocations =
(functionKind: 'hog_function' | 'hog_flow') =>
async (req: ModifiedRequest, res: express.Response): Promise<any> => {
try {
if (!this.rerunJobManager) {
return res.status(503).json({
error: 'Rerun manager not initialized (CYCLOTRON_NODE_DATABASE_URL unset)',
})
}
const { team_id, id } = req.params
const team = await this.deps.teamManager.getTeam(parseInt(team_id)).catch(() => null)
if (!team) {
return res.status(404).json({ error: 'Team not found' })
}
if (functionKind === 'hog_function') {
const hogFunction = await this.hogFunctionManager.getHogFunction(id)
if (!hogFunction || hogFunction.team_id !== team.id) {
return res.status(404).json({ error: 'Hog function not found' })
}
} else {
const hogFlow = await this.hogFlowManager.getHogFlow(id)
if (!hogFlow || hogFlow.team_id !== team.id) {
return res.status(404).json({ error: 'Hog flow not found' })
}
}
const rerunRequest = req.body as RerunRequest
const rerunJobId = await this.rerunJobManager.enqueue(team.id, functionKind, id, rerunRequest)
// Surface the wrapper job in the Invocations list immediately —
// a 'running' lifecycle row + a `rerun_queued` log line. Both
// share the same `instance_id = rerun_job_id` so the logs
// viewer in the row's expand panel picks them up automatically.
const now = new Date()
this.invocationResultsService.invocationResultsRowsService.queueRerunWrapperRow({
teamId: team.id,
parentFunctionKind: functionKind,
functionId: id,
rerunJobId,
status: 'running',
pagesProcessed: 0,
filter: rerunRequest.filter,
scheduledAt: now,
startedAt: now,
})
this.invocationResultsService.monitoringService.queueLogs(
[
{
team_id: team.id,
log_source: functionKind,
log_source_id: id,
instance_id: rerunJobId,
timestamp: DateTime.fromJSDate(now),
level: 'info',
message: `Re-run queued. Filter: ${JSON.stringify(rerunRequest.filter)}`,
},
],
functionKind
)
await this.invocationResultsService.flush()
logger.info('⚡️', 'Rerun job enqueued', {
function_kind: functionKind,
function_id: id,
team_id: team.id,
rerun_job_id: rerunJobId,
})
res.json({ rerun_job_id: rerunJobId, queued_count: 0, skipped_count: 0 })
} catch (e) {
logger.error('Error enqueueing rerun job', {
error: e instanceof Error ? e.message : String(e),
})
res.status(500).json({ error: e instanceof Error ? e.message : String(e) })
}
}
// How many of this workflow's runs are still in flight (parked on waits/delays or actively
// executing). Django calls this to show publish/edit impact before a live workflow changes.
private getHogFlowInFlightCount = async (req: ModifiedRequest, res: express.Response): Promise<any> => {
try {
if (!this.batchResolverProducer) {
return res.status(503).json({
error: 'Cyclotron producer not initialized (CYCLOTRON_NODE_DATABASE_URL unset)',
})
}
const { team_id, id } = req.params
const team = await this.deps.teamManager.getTeam(parseInt(team_id)).catch(() => null)
if (!team) {
return res.status(404).json({ error: 'Team not found' })
}
const hogFlow = await this.hogFlowManager.getHogFlow(id)
if (!hogFlow || hogFlow.team_id !== team.id) {
return res.status(404).json({ error: 'Workflow not found' })
}
const counts = await this.batchResolverProducer.countInFlightJobs(team.id, id)
return res.json({
count: counts.count,
by_action: counts.byAction,
position_unknown: counts.positionUnknown,
})
} catch (e) {
logger.error('Error counting in-flight hog flow jobs', {
error: e instanceof Error ? e.message : String(e),
})
return res.status(500).json({ error: e instanceof Error ? e.message : String(e) })
}
}
// Shared gate for the per-call scoped JWTs Django mints (reschedule, cancel): verifies the
// token and requires its claims to match the URL's team + workflow, so a leaked token can't
// touch another team or flow. Routes scoped tighter than a workflow (batch cancel) pass the
// narrower claims via extraClaims and every one must match too; a route without a workflow
// in its path (step resume) pins the job through extraClaims instead. Writes the 401 itself
// and returns false on any mismatch.
// Run a transactional email's workflow once. Django checks the template and the workflow, then calls this
// once per send. The job goes straight onto the email queue's fast lane.
//
// Auth: a per-call JWT minted by Django on its own audience and key, pinned to the team and the workflow,
// never the fleet-wide internal secret.
private postTransactionalEmail =
(schema: z.ZodType<TransactionalEmailRequest, unknown>, tokenClaims: Record<string, string | undefined>) =>
async (req: ModifiedRequest, res: express.Response): Promise<any> => {
const stages = new TransactionalEmailStageTimer()
const reply = (outcome: TransactionalEmailOutcome, status: number, body: Record<string, unknown>) => {
transactionalEmailRequestsTotal.inc({ outcome })
return res.status(status).json(body)
}
try {
stages.start('auth')
if (!this.transactionalEmailJwt.enabled) {
return reply('not_configured', 503, {
error: 'Transactional email auth not configured (WORKFLOWS_TRANSACTIONAL_EMAIL_JWT_SECRET unset)',
})
}
if (
!this.verifyScopedWorkflowJwt(
this.transactionalEmailJwt,
req,
res,
'transactional email',
tokenClaims
)
) {
transactionalEmailRequestsTotal.inc({ outcome: 'unauthorized' })
return
}
stages.start('validate')
const parsed = schema.safeParse(req.body)
if (!parsed.success) {
const issues = parsed.error.issues
// Signup accepts some addresses this check refuses, and no retry can fix an address.
if (issues.every(isRecipientFormatIssue)) {
return reply('recipient_unreachable', 422, { error: 'recipient_unreachable' })
}
// Paths and codes only: the values include the recipient's address.
logger.warn('⚠️', 'Transactional email request failed validation', {
team_id: req.params.team_id,
hog_flow_id: req.params.id,
issues: issues.map((issue) => ({ path: issue.path.join('.'), code: issue.code })),
})
return reply('invalid_request', 400, { error: parsed.error.message })
}
const { to, distinct_id: distinctId, variables, idempotency_key: idempotencyKey, send } = parsed.data
stages.start('load')
const teamLookup = this.deps.teamManager.getTeam(parseInt(req.params.team_id)).catch(() => null)
const hogFlowLookup = this.hogFlowManager.getHogFlowCheckingPrimaryOnMiss(req.params.id)
// A missing team returns before the workflow lookup is awaited, so its failure must not go unhandled.
hogFlowLookup.catch(() => undefined)
const team = await teamLookup
if (!team) {
return reply('not_found', 404, { error: 'Team not found' })
}
const hogFlow = await hogFlowLookup
if (!hogFlow || hogFlow.team_id !== team.id) {
return reply('not_found', 404, { error: 'Workflow not found' })
}
if (hogFlow.trigger?.type !== 'api') {
return reply('not_sendable', 400, { error: 'not_sendable' })
}
stages.start('build')
const event: HogFunctionInvocationGlobals['event'] = {
uuid: new UUIDT().toString(),
event: TRANSACTIONAL_EMAIL_EVENT,
distinct_id: distinctId,
timestamp: DateTime.now().toISO(),
url: '',
// The email step's recipient reads this.
properties: { to },
elements_chain: '',
}
const triggerGlobals: HogFunctionInvocationGlobals = {
event,
project: {
id: team.id,
name: team.name,
url: `${this.config.SITE_URL ?? 'http://localhost:8000'}/project/${team.id}`,
},
variables,
}
const filterGlobals = convertToHogFunctionFilterGlobal({
event,
person: undefined,
groups: {},
variables,
})
const invocation = createHogFlowInvocation(triggerGlobals, hogFlow, filterGlobals)
const files = decodeAttachments(send)
const sendToQueue = send ? queuedSend(send, files) : undefined
if (sendToQueue) {
invocation.state.send = sendToQueue
}
// The run skips the trigger pipeline that counts runs for event-triggered workflows, so the email
// worker counts it instead.
invocation.state.triggerMetricPending = true
// Straight to the email queue: the flow's only step is the email, so a hop through the
// workflow queue would only add latency to a password reset or a verification code.
const priorityClass: EmailQueuePriorityClass = 'fast'
invocation.queue = 'email'
invocation.queuePriority = EMAIL_QUEUE_PRIORITY[priorityClass]
const idempotencyKeys = this.transactionalEmailIdempotencyKeys
const fingerprint = transactionalEmailFingerprint(hogFlow.id, parsed.data, sendToQueue)
if (idempotencyKey) {
// Checked before the quota, so a retry of a send that went out gets its answer even
// when the team has hit its quota since.
stages.start('idempotency')
const reservation = await idempotencyKeys.reserve(
team.id,
idempotencyKey,
fingerprint,
invocation.id
)
switch (reservation.status) {
case 'replayed':
return reply('replayed', 202, {
status: 'queued',
invocation_id: reservation.invocationId,
replayed: true,
})
case 'in_flight':
return reply('idempotency_key_in_flight', 409, { error: 'idempotency_key_in_flight' })
case 'conflict':
return reply('idempotency_key_reused', 422, { error: 'idempotency_key_reused' })
case 'unavailable':
return reply('idempotency_unavailable', 503, { error: 'idempotency_unavailable' })
}
}
try {
// After the idempotency check, so a retry of a send accepted with files still gets its answer.
if (send?.attachments !== undefined && !this.config.CDP_EMAIL_ATTACHMENTS_ENABLED) {
if (idempotencyKey) {
stages.start('idempotency')
await idempotencyKeys.release(team.id, idempotencyKey, invocation.id)
}
return reply('invalid_request', 400, { error: fieldNotAcceptedError('attachments') })
}
stages.start('checks')
const refusal = await this.refuseTransactionalEmail(
team.id,
hogFlow,
transactionalEmailRecipients(parsed.data)
)
if (refusal) {
if (idempotencyKey) {
stages.start('idempotency')
await idempotencyKeys.release(team.id, idempotencyKey, invocation.id)
}
return reply(refusal.outcome, refusal.status, { error: refusal.outcome })
}
if (files.length) {
stages.start('files')
await this.emailAttachmentsService.store(invocation.id, team.id, files)
}
stages.start('queue_insert')
this.invocationResultsService.invocationResultsRowsService.queueLifecycleRow(invocation, 'running')
try {
await this.hogflowQueue.queueInvocations([invocation])
} catch (error) {
this.invocationResultsService.invocationResultsRowsService.dropQueuedRowsFor([invocation.id])
throw error
}
cdpEmailQueuedTotal.labels(priorityClass).inc()
} catch (error) {
// Nothing was queued, so the caller's retry with the same key must be able to send.
if (idempotencyKey) {
stages.start('idempotency')
await idempotencyKeys.release(team.id, idempotencyKey, invocation.id)
}
throw error
}
if (idempotencyKey) {
stages.start('idempotency')
await idempotencyKeys.markQueued(team.id, idempotencyKey, fingerprint, invocation.id)
}
stages.start('metrics_flush')
this.emailStatusService.queue({
teamId: team.id,
messageId: invocation.id,
status: 'queued',
recipient: to,
functionId: hogFlow.id,
})
// Best-effort rows, so the 202 waits for the hand-off to Kafka (flushed on shutdown) but not the acks.
const warnTelemetryFlushFailed = (error: unknown) =>
logger.warn('⚠️', 'Transactional email telemetry flush failed', {
error: error instanceof Error ? error.message : String(error),
invocation_id: invocation.id,
})
void this.emailStatusService.flush().catch(warnTelemetryFlushFailed)
const { produced } =
await this.invocationResultsService.invocationResultsRowsService.handOffQueuedRows()
void produced.catch(warnTelemetryFlushFailed)
return reply('queued', 202, { status: 'queued', invocation_id: invocation.id, replayed: false })
} catch (e) {
logger.error('Error queueing transactional email', {
error: e instanceof Error ? e.message : String(e),
})
return reply('error', 500, { error: e instanceof Error ? e.message : String(e) })
} finally {
stages.finish()
}
}
// The email worker skips a send for these reasons too, but only after the caller has its 202. Checking them
// here tells the caller that the email will not go out. The worker keeps its checks, because the state can
// change while the job waits on the queue.
private async refuseTransactionalEmail(
teamId: number,
hogFlow: HogFlow,
recipients: string[]
): Promise<{ outcome: TransactionalEmailOutcome; status: number } | null> {
if (hogFlow.status !== 'active') {
return { outcome: 'not_sendable', status: 400 }
}
// The lookups run at the same time, and the checks below await them in order, so the first refusal in that
// order wins. The MX check's DNS lookup runs even when an earlier check refuses, which is acceptable because
// refusals are rare and the verdict is cached per domain.
const suspension = this.teamWorkflowsConfigService.getEmailSendingSuspension(teamId)
const suppressed = Promise.all(
recipients.map((email) => this.emailSuppressionService.isSuppressed(teamId, email, { transactional: true }))
)
const quota = checkHogFlowQuotaLimits(hogFlow, teamId, this.deps.quotaLimiting)
// A DNS failure lets the send through.
const recipientSkipReasons = Promise.all(
recipients.map((email) => this.emailValidationService.getRecipientSkipReason(teamId, email))
)
// An earlier refusal returns before the later lookups are awaited, so their failures must not go unhandled.
const lookups: Promise<unknown>[] = [suspension, suppressed, quota, recipientSkipReasons]
for (const lookup of lookups) {
lookup.catch(() => undefined)
}
if (await suspension) {
return { outcome: 'sending_suspended', status: 422 }
}
if (hogFlow.email_sending_paused_at) {
return { outcome: 'sending_paused', status: 422 }
}
if ((await suppressed).some(Boolean)) {
return { outcome: 'recipient_suppressed', status: 422 }
}
if ((await quota).isLimited) {
return { outcome: 'quota_limited', status: 429 }
}
if ((await recipientSkipReasons).some(Boolean)) {
return { outcome: 'recipient_unreachable', status: 422 }
}
return null
}
private verifyScopedWorkflowJwt(
jwt: ScopedServiceJwt,
req: ModifiedRequest,
res: express.Response,
label: string,
extraClaims?: Record<string, string | undefined>
): boolean {
const { team_id, id } = req.params
const authHeader = req.headers['authorization']
const token =
typeof authHeader === 'string' && authHeader.startsWith('Bearer ') ? authHeader.slice(7) : undefined
let claims: Record<string, unknown> | undefined
try {
claims = token ? (jwt.verify(token) as typeof claims) : undefined
} catch {
claims = undefined
}
const extrasMatch = !extraClaims || Object.entries(extraClaims).every(([key, value]) => claims?.[key] === value)
const flowMatches = id === undefined || claims?.hog_flow_id === id
if (!claims || claims.team_id !== parseInt(team_id) || !flowMatches || !extrasMatch) {
res.status(401).json({ error: `Unauthorized: Invalid ${label} token` })
return false
}
return true
}
// Pull forward the wake times of this workflow's parked jobs after a timing edit. Django
// calls this (via a Celery task) when a published/saved change shortened a delay or moved a
// wait window; one call is one slice, and the caller loops with the returned bounds until
// `done`. See CyclotronV2Manager.rescheduleParkedJobs for the sweep semantics.
//
// Auth: a scoped JWT minted by Django per call, pinned to this team + workflow — NOT the
// fleet-wide internal secret (the route is exempted from that middleware). Fails closed when
// the key isn't provisioned.
private postHogFlowRescheduleParked = async (req: ModifiedRequest, res: express.Response): Promise<any> => {
try {
if (!this.batchResolverProducer) {
return res.status(503).json({
error: 'Cyclotron producer not initialized (CYCLOTRON_NODE_DATABASE_URL unset)',
})
}
if (!this.rescheduleJwt.enabled) {
return res.status(503).json({
error: 'Reschedule auth not configured (WORKFLOWS_RESCHEDULE_JWT_SECRET unset)',
})
}
const { team_id, id } = req.params
if (!this.verifyScopedWorkflowJwt(this.rescheduleJwt, req, res, 'reschedule')) {
return
}
const team = await this.deps.teamManager.getTeam(parseInt(team_id)).catch(() => null)
if (!team) {
return res.status(404).json({ error: 'Team not found' })
}
const hogFlow = await this.hogFlowManager.getHogFlow(id)
if (!hogFlow || hogFlow.team_id !== team.id) {
return res.status(404).json({ error: 'Workflow not found' })
}
const body = req.body ?? {}
const actionIds = body.action_ids
if (
!Array.isArray(actionIds) ||
actionIds.length === 0 ||
actionIds.length > 100 ||
!actionIds.every((a: unknown) => typeof a === 'string' && a.length > 0)
) {
return res.status(400).json({ error: 'action_ids must be a non-empty array of up to 100 strings' })
}
const sweepFloor = body.sweep_floor ? new Date(body.sweep_floor) : undefined
const sweepUntil = body.sweep_until ? new Date(body.sweep_until) : undefined
if ((sweepFloor && isNaN(sweepFloor.getTime())) || (sweepUntil && isNaN(sweepUntil.getTime()))) {
return res.status(400).json({ error: 'sweep_floor and sweep_until must be ISO datetimes' })
}
if (
(sweepFloor === undefined) !== (sweepUntil === undefined) ||
(sweepFloor && sweepUntil && sweepFloor >= sweepUntil)
) {
return res
.status(400)
.json({ error: 'sweep_floor and sweep_until must be passed together, with floor before until' })
}
const result = await this.batchResolverProducer.rescheduleParkedJobs({
teamId: team.id,
functionId: id,
actionIds,
sweepFloor,
sweepUntil,
})
return res.json({
swept: result.swept,
remaining: result.remaining,
done: result.done,
sweep_floor: result.sweepFloor.toISOString(),
sweep_until: result.sweepUntil.toISOString(),
})
} catch (e) {
logger.error('Error rescheduling parked hog flow jobs', {
error: e instanceof Error ? e.message : String(e),
})
return res.status(500).json({ error: e instanceof Error ? e.message : String(e) })
}
}
/**
* Wake a batch resolver parked until its realtime cohort finishes calculating. Django calls this
* once the cohort is ready. `already_resumed` is a success: a second wake for the same run is a no-op.
* Only a running resolver answers 409, so the caller retries. A finished or removed one already ran, so
* a wake retried after a lost response gets a 200 and the caller stops.
*/
private postHogFlowResumeBatchJob = async (req: ModifiedRequest, res: express.Response): Promise<any> => {
try {
if (!this.batchResolverProducer) {
return res.status(503).json({
error: 'Cyclotron producer not initialized (CYCLOTRON_NODE_DATABASE_URL unset)',
})
}
if (!this.resumeBatchJwt.enabled) {
return res.status(503).json({
error: 'Workflows scoped auth not configured (WORKFLOWS_STEP_RESUME_JWT_SECRET unset)',
})
}
const { team_id, id, batch_job_id } = req.params
// batch_job_id is pinned in the claims, so a captured token wakes exactly one run.
if (!this.verifyScopedWorkflowJwt(this.resumeBatchJwt, req, res, 'resume', { batch_job_id })) {
return
}
const team = await this.deps.teamManager.getTeam(parseInt(team_id)).catch(() => null)
if (!team) {
return res.status(404).json({ error: 'Team not found' })
}
if (batch_job_id.length > 200) {
return res.status(400).json({ error: 'batch_job_id is too long' })
}
const outcome = await this.batchResolverProducer.resumeParkedBatchResolver({
teamId: team.id,
functionId: id,
batchJobId: batch_job_id,
})
return res.status(outcome === 'job_running' ? 409 : 200).json({ outcome })
} catch (e) {
logger.error('Error resuming hog flow batch job', {
error: e instanceof Error ? e.message : String(e),
})
return res.status(500).json({ error: e instanceof Error ? e.message : String(e) })
}
}
// Wake the parked workflow step that dispatched a task run, with the run's outcome. Django
// calls this from a retrying Celery task when the run reaches a terminal status. One resume
// per call; the outcome tells the caller whether to retry (409: the worker still holds the
// job) or stop (200: delivered, or the step is past this wake).
//
// Auth mirrors the cancel routes: a per-call JWT minted by Django on its own audience and key,
// pinned to the team and to the origin key, so a leaked token can wake exactly one step.
private postWorkflowStepResume = async (req: ModifiedRequest, res: express.Response): Promise<any> => {
try {
if (!this.batchResolverProducer) {
return res.status(503).json({
error: 'Cyclotron producer not initialized (CYCLOTRON_NODE_DATABASE_URL unset)',
})
}
if (!this.stepResumeJwt.enabled) {
return res.status(503).json({
error: 'Step resume auth not configured (WORKFLOWS_STEP_RESUME_JWT_SECRET unset)',
})
}
// The token pins the origin key, so read it raw for the claim check; the body is
// validated only once the caller is allowed to wake this step.
const originKey = typeof req.body?.origin_key === 'string' ? req.body.origin_key : ''
if (!this.verifyScopedWorkflowJwt(this.stepResumeJwt, req, res, 'step resume', { origin_key: originKey })) {
return
}
const parsed = WorkflowStepResumeSchema.safeParse(req.body)
const key = parsed.success ? parseWorkflowStepDispatchKey(parsed.data.origin_key) : null
if (!parsed.success || !key) {
return res.status(400).json({ error: 'origin_key must be a workflow step dispatch key' })
}
const teamId = parseInt(req.params.team_id)
const team = await this.deps.teamManager.getTeam(teamId).catch(() => null)
if (!team) {
return res.status(404).json({ error: 'Team not found' })
}
const outcomes = await this.batchResolverProducer.resumeParkedSteps(teamId, [{ ...parsed.data, ...key }])
const outcome = outcomes.get(key.jobId) ?? 'job_missing'
return res.status(outcome === 'job_running' ? 409 : 200).json({ outcome })
} catch (e) {
logger.error('Error resuming workflow step', {
error: e instanceof Error ? e.message : String(e),
})
return res.status(500).json({ error: e instanceof Error ? e.message : String(e) })
}
}
// Flag a workflow's in-flight cyclotron jobs for cancellation. The workers own the actual
// termination (terminal status + lifecycle row + metric + log) when they observe the flag;
// this endpoint only marks rows and wakes parked ones. See CyclotronV2Manager.cancelJobs.
//
// Auth mirrors postHogFlowRescheduleParked: a per-call JWT minted by Django, pinned to this
// team + workflow, on its own audience, never the fleet-wide internal secret.
private postHogFlowCancelInvocations = async (req: ModifiedRequest, res: express.Response): Promise<any> => {
try {
if (!this.batchResolverProducer) {
return res.status(503).json({
error: 'Cyclotron producer not initialized (CYCLOTRON_NODE_DATABASE_URL unset)',
})
}
if (!this.cancelInvocationsJwt.enabled) {
return res.status(503).json({
error: 'Workflows scoped auth not configured (WORKFLOWS_CANCEL_JWT_SECRET unset)',
})
}
const { team_id, id } = req.params
if (!this.verifyScopedWorkflowJwt(this.cancelInvocationsJwt, req, res, 'cancel')) {
return
}
const team = await this.deps.teamManager.getTeam(parseInt(team_id)).catch(() => null)
if (!team) {
return res.status(404).json({ error: 'Team not found' })
}
// Deliberately no hogFlowManager lookup beyond team pinning: cancel must keep working
// for flows that were deleted with runs still parked. The JWT claims already bind the
// request to this flow id, and cancelJobs filters on (team_id, function_id).
// UUID-shaped ids only: cancelJobs binds them as ::uuid[], so a malformed id must be a
// 400 here rather than a Postgres cast error surfaced as a 500. Match the shape only:
// invocation ids are UUIDTs, whose version and variant nibbles carry a series counter
// and random bytes, so a version-aware check rejects every real id. The cap is the same
// env-driven one the Django cancel serializer validates against, so raising the env
// var lifts both sides together instead of leaving Django accepting ids this route
// then rejects with a 400 (which Django surfaces as a 500).
const maxInvocationIds = this.config.HOG_INVOCATION_RERUN_MAX_COUNT
const idsMessage = `invocation_ids must be a non-empty array of up to ${maxInvocationIds} invocation UUIDs`
const parsed = z
.object({
invocation_ids: z
.array(z.string().refine((id) => UUID.validateString(id, false), idsMessage))
.min(1, idsMessage)
.max(maxInvocationIds, idsMessage)
.optional(),
all: z.boolean().optional(),
})
.refine((body) => (body.invocation_ids !== undefined) !== (body.all === true), {
message: 'Provide exactly one of invocation_ids or all=true',
})
.safeParse(req.body ?? {})
if (!parsed.success) {
return res.status(400).json({ error: parsed.error.issues[0]?.message ?? 'Invalid request body' })
}
const { invocation_ids: invocationIds, all } = parsed.data
const result = await this.batchResolverProducer.cancelJobs({
teamId: team.id,
functionId: id,
jobIds: invocationIds,
all: all === true ? true : undefined,
// Batch-resolver jobs orchestrate audience fan-out rather than being runs;
// flagging one would silently stall a batch with no terminal reporting.
// Stopping a batch run is its own feature with its own endpoint.
excludeQueueNames: [HOGFLOW_BATCH_RESOLVE_QUEUE],
})
return res.json({
marked: result.marked,
remaining: result.remaining,
done: result.done,
})
} catch (e) {
logger.error('Error cancelling hog flow invocations', {
error: e instanceof Error ? e.message : String(e),
})
return res.status(500).json({ error: e instanceof Error ? e.message : String(e) })
}
}
// Stop a batch run: flag its resolver orchestration job and every child run for
// cancellation in one sweep — they share parent_run_id, so one selector covers both.
// The resolver's in-transaction tombstone check plus Django's repeat-until-done loop
// make this converge: once the resolver is flagged it can commit at most the one page
// that already held its row lock, and that page's children surface in the next sweep's
// remaining count.
//
// Auth mirrors the other workflows CDP calls: a per-call JWT minted by Django, pinned
// to this team + workflow, on its own audience.
private postHogFlowCancelBatchJob = async (req: ModifiedRequest, res: express.Response): Promise<any> => {
try {
if (!this.batchResolverProducer) {
return res.status(503).json({
error: 'Cyclotron producer not initialized (CYCLOTRON_NODE_DATABASE_URL unset)',
})
}
if (!this.cancelBatchJwt.enabled) {
return res.status(503).json({
error: 'Workflows scoped auth not configured (WORKFLOWS_CANCEL_JWT_SECRET unset)',
})
}
const { team_id, id, batch_job_id } = req.params
// batch_job_id is pinned in the claims too: without it, a captured token could stop
// any sibling batch of the same workflow for the token's lifetime.
if (!this.verifyScopedWorkflowJwt(this.cancelBatchJwt, req, res, 'cancel', { batch_job_id })) {
return
}
const team = await this.deps.teamManager.getTeam(parseInt(team_id)).catch(() => null)
if (!team) {
return res.status(404).json({ error: 'Team not found' })
}
if (batch_job_id.length > 200) {
return res.status(400).json({ error: 'batch_job_id is too long' })
}
const result = await this.batchResolverProducer.cancelJobs({
teamId: team.id,
functionId: id,
parentRunId: batch_job_id,
})
return res.json({
marked: result.marked,
remaining: result.remaining,
done: result.done,
})
} catch (e) {
logger.error('Error cancelling hog flow batch job', {
error: e instanceof Error ? e.message : String(e),
})
return res.status(500).json({ error: e instanceof Error ? e.message : String(e) })
}
}
private postHogFlowBatchInvocation = async (req: ModifiedRequest, res: express.Response): Promise<any> => {
try {
const { id, team_id, parent_run_id } = req.params
logger.info('⚡️', 'Received hogflow batch invocation', { id, team_id, parent_run_id })
const team = await this.deps.teamManager.getTeam(parseInt(team_id)).catch(() => null)
if (!team) {
return res.status(404).json({ error: 'Team not found' })
}
const hogFlow = await this.hogFlowManager.getHogFlow(id)
if (!hogFlow || hogFlow.team_id !== team.id) {
return res.status(404).json({ error: 'Workflow not found' })
}
if (hogFlow.trigger.type !== 'batch') {
return res.status(400).json({ error: 'Only batch Workflows are supported for batch jobs' })
}
const maxAudienceSize =
typeof req.body.max_audience_size === 'number' ? req.body.max_audience_size : undefined
if (!this.batchResolverProducer) {
throw new Error('Batch resolver producer is not configured (missing CYCLOTRON_NODE_DATABASE_URL)')
}
const realtimeCohort = parseRealtimeCohort(req.body.realtime_cohort)
if (realtimeCohort === null) {
return res.status(400).json({ error: 'realtime_cohort.wait_until must be an ISO 8601 date' })
}
const snapshotFilters: BatchResolverState['filters'] | undefined = req.body.filters
const audienceType = snapshotFilters?.audience_type ?? hogFlow.trigger.filters.audience_type
// A snapshot saved before assignment statuses existed carries assignee ids or the legacy
// flag, but no status. Resolved key by key, it inherits the live trigger's status, and
// Django rejects a status paired with assignee ids, so the run fails instead of sending
// the audience the confirm check validated. One source answers for all three keys.
const assignmentFilters =
snapshotFilters && ASSIGNMENT_FILTER_KEYS.some((key) => snapshotFilters[key] !== undefined)
? snapshotFilters
: hogFlow.trigger.filters
const assignmentStatus = assignmentFilters.assignment_status
const initialState: BatchResolverState = {
batchJobId: parent_run_id,
teamId: team.id,
hogFlowId: hogFlow.id,
filters: {
// Prefer the audience snapshot validated at dispatch time - re-reading the live
// trigger here would let an edit landing after the confirm check widen the send.
// Fallback covers callers that predate the snapshot.
audience_type: audienceType,
properties: snapshotFilters?.properties ?? (hogFlow.trigger.filters.properties || []),
filter_test_accounts:
snapshotFilters?.filter_test_accounts ??
(hogFlow.trigger.filters.filter_test_accounts || false),
tag_names: snapshotFilters?.tag_names ?? hogFlow.trigger.filters.tag_names,
assignment_status: assignmentStatus,
assigned_to_user_ids: assignmentFilters.assigned_to_user_ids,
all_roles_unassigned: assignmentStatus ? undefined : assignmentFilters.all_roles_unassigned,
},
variables: req.body.variables ?? {},
groupTypeIndex: typeof req.body.group_type_index === 'number' ? req.body.group_type_index : undefined,
// Only dedupe by email when every email action's `to` template is exactly the default
// `{{ person.properties.email }}`. Custom recipients (work_email, computed Liquid, static
// strings) would make the dedupe key diverge from the actual send target — better to skip
// dedupe than dedupe wrongly. Also skip when the flow has no email action at all.
// Account audiences carry no person (and external ids are already unique), so never dedupe.
dedupeKey: audienceType !== 'accounts' && canDedupeByEmail(hogFlow) ? ('email' as const) : undefined,
realtimeCohort,
maxAudienceSize: maxAudienceSize ?? this.config.CDP_BATCH_WORKFLOW_MAX_AUDIENCE_SIZE,
cursor: null,
totalEnqueued: 0,
pagesProcessed: 0,
attempts: 0,
startedAt: new Date().toISOString(),
}
const waitUntil = initialState.realtimeCohort?.waitUntil
await this.batchResolverProducer.createJob({
teamId: team.id,
queueName: HOGFLOW_BATCH_RESOLVE_QUEUE,
parentRunId: parent_run_id,
functionId: hogFlow.id,
state: serializeResolverState(initialState),
// A cohort that is still calculating parks the job until Django resumes it.
...(waitUntil ? { scheduled: new Date(waitUntil) } : {}),
})
if (waitUntil) {
counterBatchResolverParked.inc()
}
res.json({ status: 'queued' })
} catch (e) {
logger.error('Error handling hogflow batch invocation', { error: e })
res.status(500).json({ error: [e.message] })
}
}
private async processAndRespondToWebhook(
webhookId: string,
req: ModifiedRequest,
res: express.Response,
onSuccess: (
result: Awaited<ReturnType<typeof this.cdpSourceWebhooksConsumer.processWebhook>>
) => Promise<any> | any
): Promise<any> {
try {
const result = await this.cdpSourceWebhooksConsumer.processWebhook(webhookId, req)
if (typeof result.execResult === 'object' && result.execResult && 'httpResponse' in result.execResult) {
const httpResponse = result.execResult.httpResponse as HogFunctionWebhookResult
// Security headers to prevent XSS via content-type injection
res.set('X-Content-Type-Options', 'nosniff')
res.set('Content-Security-Policy', "default-src 'none'")
if (typeof httpResponse.body === 'string') {
const safeContentType = sanitizeContentType(
httpResponse.contentType,
httpResponse.isBase64Encoded ? 'application/octet-stream' : 'text/plain'
)
if (httpResponse.isBase64Encoded) {
const buffer = Buffer.from(httpResponse.body, 'base64')
return res.status(httpResponse.status).type(safeContentType).send(buffer)
}
return res.status(httpResponse.status).type(safeContentType).send(httpResponse.body)
} else if (typeof httpResponse.body === 'object') {
return res.status(httpResponse.status).json(httpResponse.body)
}
return res.status(httpResponse.status).send('')
}
return await onSuccess(result)
} catch (error) {
if (error instanceof SourceWebhookError) {
return res.status(error.status).json({ error: error.message })
}
logger.error('[CdpApi] Error handling webhook', { error })
return res.status(500).json({ error: 'Internal error' })
}
}
private handleWebhook =
() =>
async (req: ModifiedRequest, res: express.Response): Promise<any> => {
const { webhook_id } = req.params
return this.processAndRespondToWebhook(webhook_id, req, res, (result) => {
if (result.error) {
return res.status(500).json({ status: 'Unhandled error' })
}
if (!result.finished) {
return res.status(201).json({ status: 'queued' })
}
return res.status(200).json({ status: 'ok' })
})
}
private handleWarehouseSourceWebhook =
() =>
async (req: ModifiedRequest, res: express.Response): Promise<any> => {
const { webhook_id } = req.params
return this.processAndRespondToWebhook(webhook_id, req, res, (result) => {
if (result.error) {
return res.status(500).json({ error: 'Internal error' })
}
if (!result.finished) {
return res.status(201).json({ status: 'queued' })
}
return res.status(200).json({ status: 'ok' })
})
}
private postSesWebhook =
() =>
async (req: ModifiedRequest, res: express.Response): Promise<any> => {
try {
const { status, message } = await this.emailTrackingService.handleSesWebhook(req)
return res.status(status).json({ message })
} catch {
return res.status(500).json({ error: 'Internal error' })
}
}
private getEmailTrackingPixel =
() =>
(req: ModifiedRequest, res: express.Response): any => {
this.emailTrackingService.handleEmailTrackingPixel(req, res)
}
private getEmailTrackingRedirect =
() =>
(req: ModifiedRequest, res: express.Response): any => {
this.emailTrackingService.handleEmailTrackingRedirect(req, res)
}
private generatePreferencesToken =
() =>
(req: ModifiedRequest, res: express.Response): any => {
const { team_id, identifier } = req.body
if (!team_id || !identifier) {
return res.status(400).json({ error: 'Team ID and identifier are required' })
}
const token = this.recipientTokensService.generatePreferencesToken({
team_id,
identifier,
})
return res.status(200).json({ token })
}
private validatePreferencesToken =
() =>
(req: ModifiedRequest, res: express.Response): any => {
try {
const { token } = req.params
if (!token) {
return res.status(400).json({ error: 'Token is required' })
}
const result = this.recipientTokensService.validatePreferencesToken(token)
if (!result.valid) {
return res.status(400).json({ error: 'Invalid or expired token' })
}
return res.status(200).json({
valid: result.valid,
team_id: result.team_id,
identifier: result.identifier,
app_source_id: result.app_source_id,
instance_id: result.instance_id,
})
} catch (error) {
logger.error('[CdpApi] Error validating preferences token', error)
return res.status(500).json({ error: 'Failed to validate token' })
}
}
private handleBatchExportHogFunction =
() =>
async (req: ModifiedRequest, res: express.Response): Promise<any> => {
try {
const result = await this.batchExportHogFunctionService.execute(
{
team_id: req.params.team_id,
hog_function_id: req.params.hog_function_id,
},
req.body
)
return res.json({
status: result.error ? 'error' : 'success',
errors: result.error ? [String(result.error)] : [],
logs: result.logs,
})
} catch (e) {
if (e instanceof NotFoundError) {
return res.status(404).json({ errors: [e.message] })
} else if (e instanceof ParseError) {
return res.status(400).json({ errors: [e.message] })
} else {
console.error(e)
return res.status(500).json({ errors: [e.message] })
}
}
}
}
const buildHogExecutorAsyncOptions = (
mockAsyncFunctions: boolean,
logs: MinimalLogEntry[],
sensitiveValues?: string[]
): HogExecutorExecuteAsyncOptions => {
let mockFunctions: Record<string, (...args: any[]) => any> | undefined
if (mockAsyncFunctions) {
mockFunctions = {}
for (const name of getRegisteredAsyncFunctionNames()) {
const handler = getAsyncFunctionHandler(name)!
mockFunctions[name] = (...args: any[]) => {
const startIndex = logs.length
const result = handler.mock(args, logs)
if (sensitiveValues?.length) {
for (let i = startIndex; i < logs.length; i++) {
logs[i] = {
...logs[i],
message: sanitizeLogMessage([logs[i].message], sensitiveValues),
}
}
}
return result
}
}
}
return {
maxAsyncFunctions: MAX_ASYNC_STEPS,
asyncFunctionsNames: mockAsyncFunctions ? [] : undefined,
functions: mockFunctions,
}
}