:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { DateTime } from 'luxon'
import { HogFlow } from '~/cdp/schema/hogflow'
import type { ModifiedRequest } from '~/common/api/router'
import { instrumented } from '~/common/tracing/tracing-utils'
import { logger } from '~/common/utils/logger'
import { PromiseScheduler } from '~/common/utils/promise-scheduler'
import { UUID, UUIDT } from '~/common/utils/utils'
import { HealthCheckResult, HealthCheckResultOk, PluginsServerConfig } from '../../types'
import { createHogFlowInvocation } from '../services/hogflows/hogflow-executor.service'
import { actionIdForLogging } from '../services/hogflows/hogflow-utils'
import { IdempotencyKeys } from '../services/hogflows/idempotency-keys'
import { JobQueue } from '../services/job-queue/job-queue.interface'
import { HogWatcherFunctionState, HogWatcherState, sameWatcherState } from '../services/monitoring/hog-watcher.service'
import {
CyclotronJobInvocationHogFunction,
CyclotronJobInvocationResult,
HogFunctionFilterGlobals,
HogFunctionInvocationGlobals,
HogFunctionType,
LogEntryLevel,
MinimalAppMetric,
} from '../types'
import { logEntry } from '../utils'
import { dualRead, dualWrite } from '../utils/dual-store'
import { createInvocation, createInvocationResult } from '../utils/invocation-utils'
import { CdpConsumerBase, CdpConsumerBaseDeps } from './cdp-base.consumer'
const DISALLOWED_HEADERS = [
'x-forwarded-for',
'x-forwarded-host',
'x-forwarded-proto',
'x-forwarded-port',
'cookie',
'x-csrftoken',
'proxy-authorization',
'referer',
'forwarded',
'x-real-ip',
'true-client-ip',
]
const IDEMPOTENCY_KEY_MAX_LENGTH = 255
const getFirstHeaderValue = (value: string | string[] | undefined): string | undefined => {
return Array.isArray(value) ? value[0] : value
}
export type HogFunctionWebhookResult = {
status: number
body: Record<string, any> | string
contentType?: string
isBase64Encoded?: boolean
}
export const getCustomHttpResponse = (
result: CyclotronJobInvocationResult<CyclotronJobInvocationHogFunction>
): HogFunctionWebhookResult | null => {
if (typeof result.execResult === 'object' && result.execResult && 'httpResponse' in result.execResult) {
const httpResponse = result.execResult.httpResponse as Record<string, any>
return {
status: 'status' in httpResponse && typeof httpResponse.status === 'number' ? httpResponse.status : 500,
body: 'body' in httpResponse ? httpResponse.body : '',
contentType:
'contentType' in httpResponse && typeof httpResponse.contentType === 'string'
? httpResponse.contentType
: undefined,
isBase64Encoded:
'isBase64Encoded' in httpResponse && typeof httpResponse.isBase64Encoded === 'boolean'
? httpResponse.isBase64Encoded
: undefined,
}
}
return null
}
// Metric names a webhook template may raise from its own code. A template signals one with
// `appMetric`, beside `httpResponse`. This lets a template answer 200 to a delivery it cannot
// process, and still record the failure against the function.
const TEMPLATE_APP_METRIC_NAMES = ['missing_credential'] as const satisfies readonly MinimalAppMetric['metric_name'][]
export const getCustomAppMetricName = (
result: CyclotronJobInvocationResult<CyclotronJobInvocationHogFunction>
): MinimalAppMetric['metric_name'] | null => {
if (typeof result.execResult === 'object' && result.execResult && 'appMetric' in result.execResult) {
const appMetric = result.execResult.appMetric
if (typeof appMetric === 'string' && (TEMPLATE_APP_METRIC_NAMES as readonly string[]).includes(appMetric)) {
return appMetric as MinimalAppMetric['metric_name']
}
}
return null
}
export class SourceWebhookError extends Error {
status: number
constructor(status: number, message: string) {
super(message)
this.name = 'SourceWebhookError'
this.status = status
}
}
export class CdpSourceWebhooksConsumer extends CdpConsumerBase<PluginsServerConfig> {
protected name = 'CdpSourceWebhooksConsumer'
private hogQueue: JobQueue
private hogflowQueue: JobQueue
private promiseScheduler: PromiseScheduler
constructor(
config: PluginsServerConfig,
deps: CdpConsumerBaseDeps,
jobQueues: { hogQueue: JobQueue; hogflowQueue: JobQueue },
private idempotencyKeys: IdempotencyKeys = new IdempotencyKeys(null, 'workflow-webhook')
) {
super(config, deps)
this.promiseScheduler = new PromiseScheduler()
this.hogQueue = jobQueues.hogQueue
this.hogflowQueue = jobQueues.hogflowQueue
}
public async getWebhook(webhookId: string): Promise<{ hogFlow?: HogFlow; hogFunction: HogFunctionType } | null> {
if (!UUID.validateString(webhookId, false)) {
return null
}
// Check for hog functions
const hogFunction = await this.hogFunctionManager.getHogFunction(webhookId)
if (hogFunction?.type === 'source_webhook' && hogFunction.enabled && !hogFunction.deleted) {
return { hogFunction }
}
if (hogFunction?.type === 'warehouse_source_webhook' && hogFunction.enabled && !hogFunction.deleted) {
const templateId = hogFunction.template_id ?? 'template-warehouse-source-default'
const template = await this.hogFunctionTemplateManager.getHogFunctionTemplate(templateId)
if (template) {
hogFunction.bytecode = template.bytecode
return { hogFunction }
}
}
// Otherwise check for hog flows
const hogFlow = await this.hogFlowManager.getHogFlow(webhookId)
if (
hogFlow &&
hogFlow.status === 'active' &&
(hogFlow.trigger?.type === 'webhook' ||
hogFlow.trigger?.type === 'tracking_pixel' ||
hogFlow.trigger?.type === 'manual')
) {
const hogFunction = await this.hogFlowFunctionsService.buildHogFunction(hogFlow, hogFlow.trigger)
return { hogFlow, hogFunction }
}
return null
}
private buildRequestGlobals(hogFunction: HogFunctionType, req: ModifiedRequest): HogFunctionInvocationGlobals {
const body: Record<string, any> = req.body
const ipValue = getFirstHeaderValue(req.headers['x-forwarded-for']) || req.socket.remoteAddress || req.ip
// IP could be comma delimited list of IPs
const ips = ipValue?.split(',').map((ip) => ip.trim()) || []
const ip = ips[0]
const projectUrl = `${this.config.SITE_URL}/project/${hogFunction.team_id}`
const headers: Record<string, string> = {}
for (const [key, value] of Object.entries(req.headers)) {
const firstValue = getFirstHeaderValue(value)
if (firstValue && !DISALLOWED_HEADERS.includes(key.toLowerCase())) {
headers[key.toLowerCase()] = firstValue
}
}
const query: Record<string, string> = {}
for (const [key, value] of Object.entries(req.query)) {
const firstValue = Array.isArray(value) ? value.join(',') : value
query[key] = String(firstValue)
}
return {
source: {
id: hogFunction.id,
name: hogFunction.name ?? `Hog function: ${hogFunction.id}`,
url: `${projectUrl}/functions/${hogFunction.id}`,
},
project: {
id: hogFunction.team_id,
name: '',
url: '',
},
event: {
event: '$incoming_webhook',
properties: {},
uuid: new UUIDT().toString(),
distinct_id: req.body.distinct_id,
elements_chain: '',
timestamp: DateTime.now().toISO(),
url: '',
},
request: {
method: req.method,
headers,
ip,
body,
query,
stringBody: req.rawBody ?? '',
},
variables: req.body.$variables || {},
}
}
private getIdempotencyKey(req: ModifiedRequest, hogFlow: HogFlow): string | null {
if (hogFlow.trigger?.type !== 'webhook') {
return null
}
const idempotencyKey = getFirstHeaderValue(req.headers['idempotency-key'])
if (!idempotencyKey) {
return null
}
if (idempotencyKey.length > IDEMPOTENCY_KEY_MAX_LENGTH) {
throw new SourceWebhookError(
400,
`Idempotency-Key must be ${IDEMPOTENCY_KEY_MAX_LENGTH} characters or fewer`
)
}
return idempotencyKey
}
private async executeHogFlow(
req: ModifiedRequest,
hogFlow: HogFlow,
hogFunction: HogFunctionType
): Promise<CyclotronJobInvocationResult<CyclotronJobInvocationHogFunction>> {
const idempotencyKey = this.getIdempotencyKey(req, hogFlow)
// A key belongs to one workflow, so the same key on two workflows starts a run on each.
const scopedIdempotencyKey = idempotencyKey ? `${hogFlow.id}/${idempotencyKey}` : null
logger.info('Executing hog flow trigger', {
id: hogFlow.id,
template_id: hogFunction.template_id,
team_id: hogFlow.team_id,
})
const invocationId = new UUIDT().toString()
const triggerActionId = hogFlow.actions.find((action) => action.type === 'trigger')?.id ?? 'trigger_node'
const addLog = (level: LogEntryLevel, message: string) => {
this.hogFunctionMonitoringService.queueLogs(
[
{
team_id: hogFlow.team_id,
log_source: 'hog_flow',
log_source_id: hogFlow.id,
instance_id: invocationId,
...logEntry(level, `${actionIdForLogging({ id: triggerActionId })} ${message}`),
},
],
'hog_flow'
)
}
const addMetric = (metric: Pick<MinimalAppMetric, 'metric_kind' | 'metric_name' | 'count'>) => {
this.hogFunctionMonitoringService.queueAppMetric(
{
team_id: hogFlow.team_id,
app_source_id: hogFlow.id,
...metric,
},
'hog_flow'
)
}
try {
const globals: HogFunctionInvocationGlobals = this.buildRequestGlobals(hogFunction, req)
const globalsWithInputs = await this.hogExecutorAsync.hogExecutor.buildInputsWithGlobals(
hogFunction,
globals
)
const invocation = createInvocation(globalsWithInputs, hogFunction)
// Slightly different handling for hog flows
// Run the initial step - this allows functions not using fetches to respond immediately
const functionResult = await this.hogFlowFunctionsService.execute(invocation)
functionResult.logs.forEach((log) => addLog(log.level, log.message))
functionResult.logs = []
// Queue any queued work here. This allows us to enable delayed work like fetching eventually without blocking the API.
if (!functionResult.finished) {
throw new SourceWebhookError(500, 'Delayed processing not supported')
}
const customHttpResponse = getCustomHttpResponse(functionResult)
if (customHttpResponse) {
const level = customHttpResponse.status >= 400 ? 'warn' : 'info'
if (level === 'warn') {
const bodyStr =
typeof customHttpResponse.body === 'string'
? customHttpResponse.body
: JSON.stringify(customHttpResponse.body)
addLog(level, `Responded with response status - ${customHttpResponse.status}, reason: ${bodyStr}`)
} else {
addLog(level, `Responded with response status - ${customHttpResponse.status}`)
}
}
const capturedPostHogEvent = functionResult.capturedPostHogEvents[0]
// Add all logs to the result
if (capturedPostHogEvent) {
// For workflows, the captured event is only used as trigger data and not to actually capture the event
// Remove the execution count property to allow workflow actions to capture events without
// triggering the infinite loop protection.
const { $hog_function_execution_count, ...cleanProperties } = capturedPostHogEvent.properties || {}
// Invoke the hogflow
const triggerGlobals: HogFunctionInvocationGlobals = {
...invocation.state.globals,
event: {
...capturedPostHogEvent,
properties: cleanProperties,
uuid: new UUIDT().toString(),
elements_chain: '',
url: '',
},
}
const hogFlowInvocation = createHogFlowInvocation(
triggerGlobals,
hogFlow,
{} as HogFunctionFilterGlobals
)
hogFlowInvocation.id = invocationId // Keep the IDs consistent
if (scopedIdempotencyKey) {
// Every repeat of a key on the workflow is the same request, so the fingerprint is the workflow.
const reservation = await this.idempotencyKeys.reserve(
hogFlow.team_id,
scopedIdempotencyKey,
hogFlow.id,
invocationId
)
if (reservation.status === 'replayed') {
addLog('info', `Skipped duplicate request for Idempotency-Key "${idempotencyKey}"`)
functionResult.finished = false
functionResult.execResult = { httpResponse: { status: 200, body: { status: 'duplicate' } } }
return functionResult
}
if (reservation.status === 'in_flight') {
// The first request can still fail to queue, so this one must not report a duplicate.
functionResult.finished = false
functionResult.execResult = { httpResponse: { status: 409, body: { status: 'in_flight' } } }
return functionResult
}
if (reservation.status === 'unavailable') {
// A webhook run matters more than its deduplication, so the run queues without a claim.
logger.warn('Idempotency store unavailable, queueing the hog flow run anyway', {
team_id: hogFlow.team_id,
hog_flow_id: hogFlow.id,
})
}
}
addMetric({
metric_kind: 'other',
metric_name: 'triggered',
count: 1,
})
// Queued before queueInvocations serializes the invocation, because
// queueLifecycleRow stamps `state.firstScheduledAt` and only a stamp set before
// serialization reaches cyclotron.
this.invocationResultsService.invocationResultsRowsService.queueLifecycleRow(
hogFlowInvocation,
'running'
)
try {
await this.hogflowQueue.queueInvocations([hogFlowInvocation])
} catch (error) {
if (scopedIdempotencyKey) {
await this.idempotencyKeys.release(hogFlow.team_id, scopedIdempotencyKey, invocationId)
}
throw error
}
if (scopedIdempotencyKey) {
await this.idempotencyKeys.markQueued(
hogFlow.team_id,
scopedIdempotencyKey,
hogFlow.id,
invocationId
)
}
} else {
addMetric({
metric_kind: 'failure',
metric_name: 'trigger_failed',
count: 1,
})
}
// Always set to false for hog flows as this triggers the flow to continue so we dont want metrics for this
functionResult.finished = false
return functionResult
} catch (error) {
logger.error('Error triggering hog flow', { error })
// The 'running' row is queued before the invocation reaches cyclotron, so a throw after
// that point leaves a row for a run that does not exist and would never terminate.
this.invocationResultsService.invocationResultsRowsService.dropQueuedRowsFor([invocationId])
addMetric({
metric_kind: 'failure',
metric_name: 'trigger_failed',
count: 1,
})
addLog('error', `Error triggering flow: ${error.message}`)
// NOTE: We only return a hog function result. We track out own logs and errors here
return createInvocationResult(
createInvocation({} as any, hogFunction),
{},
{
finished: false,
error: error.message,
}
)
}
}
private async executeHogFunction(
req: ModifiedRequest,
hogFunction: HogFunctionType,
hogFunctionState: HogWatcherFunctionState | null
): Promise<CyclotronJobInvocationResult<CyclotronJobInvocationHogFunction>> {
let result: CyclotronJobInvocationResult<CyclotronJobInvocationHogFunction>
try {
const globals: HogFunctionInvocationGlobals = this.buildRequestGlobals(hogFunction, req)
const globalsWithInputs = await this.hogExecutorAsync.hogExecutor.buildInputsWithGlobals(
hogFunction,
globals
)
const invocation = createInvocation(globalsWithInputs, hogFunction)
if (hogFunctionState?.state === HogWatcherState.degraded) {
// Degraded functions are not executed immediately
invocation.queue = 'hogoverflow'
await this.hogQueue.queueInvocations([invocation])
result = createInvocationResult<CyclotronJobInvocationHogFunction>(
invocation,
{},
{
finished: false,
logs: [
{
level: 'warn',
message: 'Function scheduled for future execution due to degraded state',
timestamp: DateTime.now(),
},
],
}
)
result.execResult = {
// TODO: Add support for a default response as an input
httpResponse: {
status: 200,
body: '',
},
}
} else {
// Run the initial step - this allows functions not using fetches to respond immediately
result = await this.hogExecutorAsync.execute(invocation)
// Queue any queued work here. This allows us to enable delayed work like fetching eventually without blocking the API.
if (!result.finished) {
await this.hogQueue.queueInvocationResults([result])
}
const customHttpResponse = getCustomHttpResponse(result)
const customAppMetricName = getCustomAppMetricName(result)
if (customAppMetricName) {
result.metrics.push({
team_id: hogFunction.team_id,
app_source_id: hogFunction.id,
metric_kind: 'failure',
metric_name: customAppMetricName,
count: 1,
})
}
if (customHttpResponse) {
// A template that drops a delivery answers 200, to stop the provider retrying.
// The status alone therefore no longer shows whether we processed the delivery.
const level = customHttpResponse.status >= 400 || customAppMetricName ? 'warn' : 'info'
if (level === 'warn') {
const bodyStr =
typeof customHttpResponse.body === 'string'
? customHttpResponse.body
: JSON.stringify(customHttpResponse.body)
result.logs.push(
logEntry(
level,
`Responded with response status - ${customHttpResponse.status}, reason: ${bodyStr}`
)
)
} else {
result.logs.push(
logEntry(level, `Responded with response status - ${customHttpResponse.status}`)
)
}
}
}
} catch (error) {
logger.error('Error executing hog function', { error })
result = createInvocationResult(
createInvocation({} as any, hogFunction),
{},
{
finished: true,
error: error.message,
logs: [{ level: 'error', message: error.message, timestamp: DateTime.now() }],
}
)
}
await this.invocationResultsService.queueInvocationResults([result])
return result
}
@instrumented('cdpSourceWebhooksConsumer.processWebhook')
public async processWebhook(
identifier: string,
req: ModifiedRequest
): Promise<CyclotronJobInvocationResult<CyclotronJobInvocationHogFunction>> {
// NOTE: To simplify usage we allow setting a range of extensions for webhooks
// Currently we just ignore it
const [webhookId, _extension] = identifier.split('.')
const [webhook, hogFunctionState] = await Promise.all([
this.getWebhook(webhookId),
dualRead(
'hog-watcher.getCachedEffectiveState',
() => this.hogWatcher.getCachedEffectiveState(webhookId),
() => this.hogWatcherMirror.getCachedEffectiveState(webhookId),
sameWatcherState
),
])
if (!webhook) {
throw new SourceWebhookError(404, 'Not found')
}
const { hogFunction, hogFlow } = webhook
if (hogFunctionState?.state === HogWatcherState.disabled) {
this.hogFunctionMonitoringService.queueAppMetric(
{
team_id: hogFunction.team_id,
app_source_id: hogFunction.id,
metric_kind: 'failure',
metric_name: 'disabled_permanently',
count: 1,
},
hogFlow ? 'hog_flow' : 'hog_function'
)
throw new SourceWebhookError(429, 'Disabled')
}
const result = hogFlow
? await this.executeHogFlow(req, hogFlow, hogFunction)
: await this.executeHogFunction(req, hogFunction, hogFunctionState)
void this.promiseScheduler.schedule(
this.invocationResultsService.flush(),
dualWrite(
'hog-watcher.observeResultsBuffered',
() => this.hogWatcher.observeResultsBuffered(result),
() => this.hogWatcherMirror.observeResultsBuffered(result)
)
)
return result
}
public override async start(): Promise<void> {
await super.start()
// Make sure we are ready to produce to cyclotron first
await Promise.all([this.hogQueue.startAsProducer(), this.hogflowQueue.startAsProducer()])
}
public override async stop(): Promise<void> {
await Promise.all([this.hogQueue.stopProducer(), this.hogflowQueue.stopProducer()])
await this.promiseScheduler.waitForAllSettled()
// IMPORTANT: super always comes last
await super.stop()
}
public isHealthy(): HealthCheckResult {
// TODO: What should we consider healthy / unhealthy here? kafka?
return new HealthCheckResultOk()
}
}