:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { DateTime } from 'luxon'
import { instrumented } from '~/common/tracing/tracing-utils'
import { parseJSON } from '~/common/utils/json-parse'
import { logger } from '~/common/utils/logger'
import { FetchOptions } from '~/common/utils/request'
import { TeamManager } from '~/common/utils/team-manager'
import { getAsyncFunctionHandler, getRegisteredAsyncFunctionNames } from '../async-function-registry'
import '../async-functions'
import type {
CyclotronJobInvocationHogFunction,
CyclotronJobInvocationResult,
HogFunctionType,
MinimalAppMetric,
MinimalLogEntry,
} from '../types'
import { createAddLogFunction, destinationE2eLagMsSummary } from '../utils'
import { resolveAwsSigV4Credentials, signAwsRequest } from '../utils/aws-sigv4'
import {
MAX_FETCH_RESPONSE_BYTES,
cdpTrackedFetch,
fetchErrorDetail,
isFetchResponseRetriable,
} from '../utils/cdp-fetch'
import { createInvocationResult } from '../utils/invocation-utils'
import { isNonFailureStatus } from '../utils/non-failure-status-codes'
import { ScopedServiceJwt } from '../utils/scoped-service-jwt'
import { mergeSecretHeaders, resolveSecretHeaders } from '../utils/secret-headers'
import { resolveStandardWebhooksKey, signStandardWebhooksRequest } from '../utils/standard-webhooks'
import { HogExecutorExecuteOptions, HogExecutorPreviousResult, HogExecutorService } from './hog-executor.service'
import { HogInputsService } from './hog-inputs.service'
import {
EMAIL_QUEUE_PRIORITY,
cdpEmailQueuedTotal,
getEmailQueuePriority,
getEmailQueuePriorityClass,
} from './messaging/email-priority'
import { EmailService } from './messaging/email.service'
import { PushNotificationService } from './messaging/push-notification.service'
import { RecipientTokensService, preferencesTokenSourceForInvocation } from './messaging/recipient-tokens.service'
import {
SELF_LOOP_MAX_DEPTH,
getSelfLoopDepth,
injectSelfLoopDepth,
isPostHogIngestUrl,
isSelfReferentialIngestFetch,
selfLoopGuardCounter,
} from './self-loop-guard'
export interface HogExecutorAsyncConfig {
googleAdwordsDeveloperToken: string
fetchRetries: number
fetchBackoffBaseMs: number
fetchBackoffMaxMs: number
siteUrl: string
internalApiBaseUrl: string
}
/**
* Every capability the async functions can reach is required - an async executor missing one is a
* misconfiguration, not a supported mode, and would only surface as a runtime throw deep inside a
* customer's function. Callers that don't need any of them want HogExecutorService directly.
*/
export interface HogExecutorAsyncDependencies {
teamManager: TeamManager
conversationsTicketsJwt: ScopedServiceJwt
customerAnalyticsAccountsJwt: ScopedServiceJwt
hogInputsService: HogInputsService
emailService: EmailService
recipientTokensService: RecipientTokensService
pushNotificationService: PushNotificationService
}
export type HogExecutorExecuteAsyncOptions = HogExecutorExecuteOptions & {
maxAsyncFunctions?: number
maxFetchRetries?: number
// Set only by the editor's test panel ("Run test"), marking this invocation as a test send. Two
// effects: emails run inline via EmailService instead of routing to the email queue (the test
// endpoint executes in-process and never enqueues to cyclotron, so routing would leave the job
// unworked), and messaging channels skip metrics/assets so a test doesn't pollute the workflow's
// Metrics and Assets tabs.
isTest?: boolean
}
/**
* Hog execution plus everything a function needs to suspend and resume: fetches, emails, push
* notifications, and the queue routing between the workers that service them.
*
* The synchronous Hog core is exposed as `hogExecutor` rather than re-wrapped, so callers reach
* `buildInputsWithGlobals` / `buildHogFunctionInvocations` / `getSensitiveValues` on the thing that
* actually owns them.
*/
export class HogExecutorAsyncService {
constructor(
public readonly hogExecutor: HogExecutorService,
private config: HogExecutorAsyncConfig,
private deps: HogExecutorAsyncDependencies
) {}
@instrumented('hog-executor.executeWithAsyncFunctions')
async executeWithAsyncFunctions(
invocation: CyclotronJobInvocationHogFunction,
options?: HogExecutorExecuteAsyncOptions
): Promise<CyclotronJobInvocationResult<CyclotronJobInvocationHogFunction>> {
let asyncFunctionCount = 0
const maxAsyncFunctions = options?.maxAsyncFunctions ?? 1
// Shared with inline async handlers (see AsyncFunctionContext.consumeInlineAsyncBudget):
// they do real network I/O without ever setting queueParameters, so the queued-type
// counting below never sees them. Without this they could run unbounded up to the VM's
// own step cap, each holding this worker slot for the full retry round instead of
// rescheduling like a queued fetch does.
const consumeInlineAsyncBudget = (): void => {
asyncFunctionCount++
if (asyncFunctionCount > maxAsyncFunctions) {
throw new Error(
`Max async functions reached: ${maxAsyncFunctions}. This function performed too many API calls in a single execution.`
)
}
}
let result: CyclotronJobInvocationResult<CyclotronJobInvocationHogFunction> | null = null
const metrics: MinimalAppMetric[] = []
const logs: MinimalLogEntry[] = []
while (!result || !result.finished) {
const nextInvocation: CyclotronJobInvocationHogFunction = result?.invocation ?? invocation
const queueParamsType = nextInvocation.queueParameters?.type
if (['fetch', 'sendPushNotification', 'email'].includes(queueParamsType ?? '')) {
asyncFunctionCount++
if (result && asyncFunctionCount > maxAsyncFunctions) {
// We don't want to block the consumer too much hence we have a limit on async functions
logger.debug('🦔', `[HogExecutor] Max async functions reached: ${maxAsyncFunctions}`)
break
}
// Queue-aware routing: each worker can execute some actions inline
// and routes others to a specialized queue. The email worker sends
// emails inline but routes fetches back to hogflow. The hogflow
// worker does fetches inline but routes emails to the email queue.
//
// Future: once we add an execution time budget, the email worker
// will also handle fetches inline. The only reason to reschedule
// back to hogflow will be when overall execution time exceeds the
// budget, to avoid blocking the queue.
if (queueParamsType === 'fetch') {
if (invocation.queue === 'email') {
// Intermediate results clone away queueMetadata (createInvocationResult
// drops it unless the target queue is passed explicitly), so read the
// stash through the entry invocation too — it still carries the row's copy.
result = this.routeToQueue(
nextInvocation,
nextInvocation.queueMetadata?.originQueue ??
invocation.queueMetadata?.originQueue ??
'hogflow',
nextInvocation.queueMetadata?.originPriority ??
invocation.queueMetadata?.originPriority ??
invocation.queuePriority
)
} else {
result = await this.executeFetch(nextInvocation, options)
}
} else if (queueParamsType === 'sendPushNotification') {
result = await this.deps.pushNotificationService.executeSendPushNotification(
nextInvocation,
options?.isTest ?? false
)
if (invocation.queue === 'email' && !result.finished) {
this.keepPushRetryInEmailQueueOrder(result, invocation)
}
} else if (queueParamsType === 'email') {
// Route to the email queue unless this is a test run: tests execute in-process and
// never enqueue, so routing would leave the job unworked.
const routeToEmailQueue = invocation.queue !== 'email' && !options?.isTest
if (routeToEmailQueue) {
// Stash the entry invocation's priority as the origin, not nextInvocation's:
// an earlier execute()/executeFetch() in this loop cloned the invocation and
// reset its queuePriority to 0, so reading nextInvocation here would restore 0
// on return and jump the run to the front of the origin queue.
result = this.routeEmailToQueue(nextInvocation, invocation.queuePriority)
} else {
// A flow already on the email queue sends inline, so this send never went
// through routeEmailToQueue's classification. Refresh the priority from
// the current action's metadata so a throttle retry re-queues under this
// send's class rather than the previous email action's.
if (invocation.queue === 'email') {
nextInvocation.queueMetadata = invocation.queueMetadata
nextInvocation.queuePriority = getEmailQueuePriority(nextInvocation.hogFunction.metadata)
}
// isTest is forwarded so a test send stays out of the email's engagement tracking.
result = await this.deps.emailService.executeSendEmail(nextInvocation, options?.isTest ?? false)
}
} else {
throw new Error(`Unknown queue type: ${queueParamsType}`)
}
} else {
// Finish execution, carrying forward previous execResult
// Tricky: We don't pass metrics in previousResult as they're accumulated in the local metrics array
const { metrics: _m, logs: _l, ...previousResultWithoutMetrics } = result || {}
result = await this.execute(
nextInvocation,
options,
previousResultWithoutMetrics,
consumeInlineAsyncBudget
)
}
logs.push(...result.logs)
metrics.push(...result.metrics)
// If we have finished _or_ something has been scheduled to run later _or_ the job was routed to a different queue then we break the loop
if (result.finished || result.invocation.queueScheduledAt || result.invocation.queue !== invocation.queue) {
break
}
}
if (result.finished) {
const capturedAt = invocation.state.globals.event?.captured_at
if (capturedAt) {
const e2eLagMs = Date.now() - new Date(capturedAt).getTime()
destinationE2eLagMsSummary.observe(e2eLagMs)
}
}
result.logs = logs
result.metrics = metrics
return result
}
/**
* A single Hog step with async functions available: the program can call `fetch`, `sendEmail`
* and friends, and its handler leaves the queue parameters the resumed run needs.
*/
async execute(
invocation: CyclotronJobInvocationHogFunction,
options: HogExecutorExecuteOptions = {},
previousResult: HogExecutorPreviousResult = {},
// Callers outside executeWithAsyncFunctions' loop (e.g. the source-webhooks consumer)
// run at most one async step per execute() call already, so a no-op budget is safe there.
consumeInlineAsyncBudget: () => void = () => {}
): Promise<CyclotronJobInvocationResult<CyclotronJobInvocationHogFunction>> {
return this.hogExecutor.execute(
invocation,
{
...options,
asyncFunctionsNames: options.asyncFunctionsNames ?? getRegisteredAsyncFunctionNames(),
functions: {
generateMessagingPreferencesUrl: (identifier): string | null => {
return identifier && typeof identifier === 'string'
? this.deps.recipientTokensService.generatePreferencesUrl(
{ team_id: invocation.teamId, identifier },
preferencesTokenSourceForInvocation(invocation)
)
: null
},
...options.functions,
},
onAsyncFunction: async ({ name, args, globals }, result) => {
const handler = getAsyncFunctionHandler(name)
if (!handler) {
throw new Error(`Unknown async function '${name}'`)
}
// Async handlers are responsible for ensuring the resumed VM stack contains
// their return value before it next runs - either by pushing directly onto
// result.invocation.state.vmState.stack (synchronous handlers) or by deferring
// the push to executeFetch / executeSendEmail (queueing handlers). See the
// RETURN-VALUE CONTRACT comment in cdp/async-functions/example.ts.
await handler.execute(
args,
{
invocation: result.invocation,
globals,
teamManager: this.deps.teamManager,
siteUrl: this.config.siteUrl,
internalApiBaseUrl: this.config.internalApiBaseUrl,
conversationsTicketsJwt: this.deps.conversationsTicketsJwt,
customerAnalyticsAccountsJwt: this.deps.customerAnalyticsAccountsJwt,
consumeInlineAsyncBudget,
},
result
)
},
},
previousResult
)
}
/**
* Routes an email send to the dedicated email queue instead of sending inline.
* The email worker will pick this up, send via SES, and return the job to the
* original queue so the workflow can continue.
*/
private routeEmailToQueue(
invocation: CyclotronJobInvocationHogFunction,
originPriority: number
): CyclotronJobInvocationResult<CyclotronJobInvocationHogFunction> {
const priorityClass = getEmailQueuePriorityClass(invocation.hogFunction.metadata)
const result = createInvocationResult<CyclotronJobInvocationHogFunction>(
invocation,
{
queue: 'email',
// The email queue dequeues transactional-class sends ahead of bulk ones;
// originPriority is stashed so routeToQueue can restore the pre-email
// priority when the job returns to its origin queue, keeping the email
// classes out of the hogflow queue's ordering.
queuePriority: EMAIL_QUEUE_PRIORITY[priorityClass],
queueParameters: invocation.queueParameters,
queueMetadata: {
...invocation.queueMetadata,
originQueue: invocation.queue,
originPriority,
},
},
{ finished: false }
)
result.metrics.push({
team_id: invocation.teamId,
app_source_id: invocation.parentRunId ?? invocation.functionId,
instance_id: invocation.state.actionId || invocation.id,
metric_kind: 'email',
metric_name: 'email_queued',
count: 1,
})
cdpEmailQueuedTotal.labels(priorityClass).inc()
return result
}
private keepPushRetryInEmailQueueOrder(
result: CyclotronJobInvocationResult<CyclotronJobInvocationHogFunction>,
entryInvocation: CyclotronJobInvocationHogFunction
): void {
const { originQueue, originPriority } = entryInvocation.queueMetadata ?? {}
result.invocation.queueMetadata = { ...result.invocation.queueMetadata, originQueue, originPriority }
result.invocation.queuePriority = getEmailQueuePriority(result.invocation.hogFunction.metadata)
}
private routeToQueue(
invocation: CyclotronJobInvocationHogFunction,
targetQueue: string,
originPriority: number
): CyclotronJobInvocationResult<CyclotronJobInvocationHogFunction> {
return createInvocationResult<CyclotronJobInvocationHogFunction>(
invocation,
{
queue: targetQueue as CyclotronJobInvocationHogFunction['queue'],
// Restore the priority the job had before routeEmailToQueue reclassified
// it, so an email-class value never orders jobs on the origin queue.
queuePriority: originPriority,
queueParameters: invocation.queueParameters,
queueMetadata: undefined,
},
{ finished: false }
)
}
private async loadIntegrationTokenReplacements(hogFunction: HogFunctionType): Promise<[string, string][]> {
const integrationInputs = await this.deps.hogInputsService.loadIntegrationInputs(hogFunction)
return Object.values(integrationInputs).flatMap(({ value }): [string, string][] => {
// integration_multi inputs resolve to an array of integrations (e.g. push channels)
// and do not take part in the single access-token placeholder substitution.
if (Array.isArray(value) || !value?.access_token_raw) {
return []
}
return this.deps.hogInputsService
.acceptedAccessTokenPlaceholders(value.$integration_id)
.map((placeholder) => [placeholder, value.access_token_raw])
})
}
@instrumented('hog-executor.executeFetch')
async executeFetch(
invocation: CyclotronJobInvocationHogFunction,
options?: Pick<HogExecutorExecuteAsyncOptions, 'maxFetchRetries'>
): Promise<CyclotronJobInvocationResult<CyclotronJobInvocationHogFunction>> {
const templateId = invocation.hogFunction.template_id ?? 'unknown'
if (invocation.queueParameters?.type !== 'fetch') {
throw new Error('Bad invocation')
}
const params = invocation.queueParameters
const result = createInvocationResult<CyclotronJobInvocationHogFunction>(
invocation,
{},
{
finished: false,
}
)
const addLog = createAddLogFunction(result.logs)
const method = params.method.toUpperCase()
let headers = params.headers ?? {}
if (params.url.startsWith('https://googleads.googleapis.com/') && !headers['developer-token']) {
headers['developer-token'] = this.config.googleAdwordsDeveloperToken
}
const tokenReplacements = await this.loadIntegrationTokenReplacements(invocation.hogFunction)
const replaceTokens = (value: string): string =>
tokenReplacements.reduce((acc, [placeholder, token]) => acc.replaceAll(placeholder, token), value)
params.body = params.body ? replaceTokens(params.body) : params.body
headers = Object.fromEntries(Object.entries(headers).map(([key, value]) => [key, replaceTokens(value)]))
params.url = replaceTokens(params.url)
// Bound event-forwarding loops: a fetch back into this project's own ingestion
// endpoint re-enters the pipeline and can re-trigger this same function. The
// ingest-URL check gates the team lookup so external fetches (the common case) pay
// nothing, and the whole block fails open - the guard must never break a destination
// it was only meant to protect.
if (isPostHogIngestUrl(params.url)) {
try {
const team = await this.deps.teamManager.getTeam(invocation.teamId)
if (team && isSelfReferentialIngestFetch({ url: params.url, body: params.body, team })) {
// Depth is counted per function id, so this destination is bounded only
// by how many times IT has re-fed itself - an event that merely passed
// through other functions can never trip the guard for it.
const functionId = invocation.hogFunction.id
const depth = getSelfLoopDepth(invocation.state.globals.event?.properties, functionId)
if (depth >= SELF_LOOP_MAX_DEPTH) {
// This destination has re-fed itself to the cap - break it.
selfLoopGuardCounter.inc({ mode: 'enforce', action: 'blocked' })
addLog(
'error',
`Refusing to fetch a PostHog ingestion endpoint using this project's own API key - this destination's event-forwarding loop has already repeated ${SELF_LOOP_MAX_DEPTH} times. To capture an event back into this project use the 'postHogCapture' helper, or to enrich incoming events use a transformation.`
)
result.error = new Error('Self-referential event-forwarding loop blocked at max depth')
result.finished = true
return result
}
// Under the cap - stamp this destination's next hop and proceed.
selfLoopGuardCounter.inc({ mode: 'enforce', action: 'allowed_with_counter' })
params.body = injectSelfLoopDepth(params.body, functionId, depth + 1)
}
} catch (err) {
logger.warn('🦔', '[HogExecutor] Self-loop guard skipped due to an internal error', {
error: err,
teamId: invocation.teamId,
})
}
}
// AWS SigV4 signatures expire after ~5 minutes. Sign immediately before the
// fetch (every attempt — including retries) so a request that sat in the
// backoff queue or whose first attempt timed out cannot reach AWS with a
// stale signature. Signing artifacts (Authorization, X-Amz-Date) are
// regenerated here and never persisted back to queueParameters. Credential
// resolution + missing-input handling live in `aws-sigv4.ts` — see
// `resolveAwsSigV4Credentials` for the encrypted_inputs/inputs lookup order.
const failSigning = (error: string): CyclotronJobInvocationResult<CyclotronJobInvocationHogFunction> => {
addLog('error', error)
result.error = new Error(error)
result.finished = true
return result
}
let signedHeaders = headers
// Secret headers are merged before signing so that the signatures cover them, and they are never written
// back to queueParameters, so every attempt resolves them again from the hog function.
if (params.secret_headers_input) {
const resolved = resolveSecretHeaders(params.secret_headers_input, invocation.hogFunction)
if (!resolved.ok) {
return failSigning(resolved.error)
}
signedHeaders = mergeSecretHeaders(signedHeaders, resolved.headers)
}
if (params.aws_sigv4) {
const resolved = resolveAwsSigV4Credentials(params.aws_sigv4, invocation.hogFunction)
if (!resolved.ok) {
return failSigning(resolved.error)
}
signedHeaders = signAwsRequest({
method,
url: params.url,
body: params.body ?? '',
headers: signedHeaders,
credentials: resolved.credentials,
})
}
// Standard Webhooks signatures embed a timestamp the receiver checks
// against a tolerance window (5 minutes in the reference libraries), so
// like AWS SigV4 above they are computed immediately before each attempt
// and never persisted back to queueParameters. The `webhook-id` the
// receiver dedupes on is the exception: it comes from the queue payload,
// which the retry path preserves, so it stays constant across attempts.
if (params.standard_webhooks) {
const resolved = resolveStandardWebhooksKey(params.standard_webhooks, invocation.hogFunction)
if (!resolved.ok) {
return failSigning(resolved.error)
}
signedHeaders = signStandardWebhooksRequest({
webhookId: params.standard_webhooks.webhook_id,
body: params.body ?? '',
headers: signedHeaders,
key: resolved.key,
})
}
const fetchParams: FetchOptions = { method, headers: signedHeaders }
if (!['GET', 'HEAD'].includes(method) && params.body) {
fetchParams.body = params.body
}
const { fetchError, fetchResponse, fetchDuration } = await cdpTrackedFetch({
url: params.url,
fetchParams,
templateId,
teamId: invocation.teamId,
hogFunctionId: invocation.hogFunction.id,
})
result.invocation.state.timings.push({
kind: 'async_function',
duration_ms: fetchDuration,
})
result.invocation.state.attempts++
if (!fetchResponse || (fetchResponse?.status && fetchResponse.status >= 400)) {
const nonFailureSchemaEntry = invocation.hogFunction.inputs_schema?.find(
(s) => s.type === 'non_failure_status_codes'
)
const nonFailureConfig = nonFailureSchemaEntry
? (invocation.hogFunction.inputs?.[nonFailureSchemaEntry.key]?.value as
| Array<number | string>
| null
| undefined)
: undefined
const isNonFailure = isNonFailureStatus(fetchResponse?.status, nonFailureConfig)
const backoffMs = Math.min(
this.config.fetchBackoffBaseMs * result.invocation.state.attempts +
Math.floor(Math.random() * this.config.fetchBackoffBaseMs),
this.config.fetchBackoffMaxMs
)
const canRetry = isFetchResponseRetriable(fetchResponse, fetchError)
const maxRetries = options?.maxFetchRetries ?? this.config.fetchRetries
// `canRetry` only says the failure class is retriable. On the last attempt it is still
// true while no retry follows, so the customer-facing log has to gate on the same
// condition the scheduling below does.
const willRetry = canRetry && result.invocation.state.attempts < maxRetries
let message = `HTTP fetch failed on attempt ${result.invocation.state.attempts} with status code ${
fetchResponse?.status ?? '(none)'
}.`
if (fetchError) {
message += ` Error: ${fetchErrorDetail(fetchError)}.`
}
if (willRetry) {
message += ` Retrying.`
}
addLog(isNonFailure ? 'info' : 'error', message)
if (willRetry) {
await fetchResponse?.dump()
result.invocation.queueParameters = params
result.invocation.queuePriority = invocation.queuePriority + 1
result.invocation.queueScheduledAt = DateTime.utc().plus({ milliseconds: backoffMs })
return result
} else if (!isNonFailure) {
result.error = new Error(message)
}
}
// Reset the attempts as we are done
result.invocation.state.attempts = 0
let body: unknown = undefined
try {
body = await fetchResponse?.text()
if (typeof body === 'string') {
try {
body = parseJSON(body)
} catch {
// Pass through the error
}
}
} catch (e) {
addLog(
'error',
e.name === 'ResponseTooLargeError'
? `The response body is larger than ${MAX_FETCH_RESPONSE_BYTES / 1024 / 1024} MB, so it was not read.`
: `Failed to parse response body: ${e.message}`
)
body = undefined
}
// Keep the status 500 fallback so template guards like `if (res.status
// >= 400) throw` still fire on a client-side abort and prevent
// subsequent fetches in multi-step templates from running against
// broken assumptions. Populate body with the real client-side error
// name and message so the terminal log stops reading as "status 500"
// with an empty body and instead tells the customer what actually
// happened (connect timeout, socket abort, DNS failure, etc.).
const hogVmResponse: {
status: number
body: unknown
} = {
status: fetchResponse?.status ?? 500,
body: body ?? (fetchError ? `${fetchError.name}: ${fetchErrorDetail(fetchError)}` : undefined),
}
// Finally we create the response object as the VM expects
result.invocation.state.vmState!.stack.push(hogVmResponse)
result.execResult = hogVmResponse
result.metrics.push({
team_id: invocation.teamId,
app_source_id: invocation.parentRunId ?? invocation.functionId,
metric_kind: 'other',
metric_name: 'fetch',
count: 1,
})
return result
}
}