Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

21.0 KB
Raw Blame
import { HogVMErrorKind, HogVMException, convertHogToJS } from '@posthog/hogvm'

import { CyclotronInputType } from '~/cdp/schema/cyclotron'
import { ACCESS_TOKEN_PLACEHOLDER } from '~/common/config/constants'
import { logger } from '~/common/utils/logger'

import { HogFunctionInvocationGlobals, HogFunctionInvocationGlobalsWithInputs, HogFunctionType } from '../types'
import { EncryptedFields } from '../utils/encryption-utils'
import { HogConfigurationError, isHogVMErrorKind, withBytecodeContract } from '../utils/hog-error-classification'
import { execHog } from '../utils/hog-exec'
import { LiquidRenderBudget, LiquidRenderer } from '../utils/liquid'
import { NATIVE_PUSH_TEMPLATE_ID, getDevicePushSubscriptionToken } from '../utils/push-subscription-utils'
import { IntegrationManagerService } from './managers/integration-manager.service'
import { PreferencesTokenSource, RecipientTokensService } from './messaging/recipient-tokens.service'

export const EXTEND_OBJECT_KEY = '$$_extend_object'

const EMAIL_INPUT_TYPES = ['native_email', 'email']

// Email fields that become headers or the plain-text part, where an HTML entity shows up literally.
// `html` and `preheader` are not here: both are written into the HTML body, so their output stays escaped.
const EMAIL_PLAIN_TEXT_FIELDS = [
    'subject',
    'text',
    'replyTo',
    'cc',
    'bcc',
    'to.email',
    'to.name',
    'from.email',
    'from.name',
]

const ACCESS_TOKEN_PLACEHOLDER_PURPOSE = 'integration-access-token-placeholder'

const accessTokenPlaceholderBase = (integrationId: number): string => `${ACCESS_TOKEN_PLACEHOLDER}${integrationId}`

const signedAccessTokenPlaceholder = (base: string, signature: string): string => `${base}_${signature.slice(0, 32)}`

export class HogInputsService {
    constructor(
        private integrationManager: IntegrationManagerService,
        private recipientTokensService: RecipientTokensService | undefined,
        private encryptedFields: EncryptedFields
    ) {}

    public async buildInputs(
        hogFunction: HogFunctionType,
        globals: HogFunctionInvocationGlobals,
        additionalInputs?: Record<string, any>,
        preferencesTokenSource?: PreferencesTokenSource
    ): Promise<Record<string, any>> {
        // TODO: Load the values from the integrationManager
        const newGlobals: HogFunctionInvocationGlobalsWithInputs = {
            ...globals,
            inputs: {},
        }
        const ownInputs: HogFunctionType['inputs'] = {
            // Include the inputs from the hog function
            ...hogFunction.inputs,
            ...hogFunction.encrypted_inputs,
            // Plus any additional inputs
            ...additionalInputs,
        }
        const inputs: HogFunctionType['inputs'] = {
            ...ownInputs,
            // and decode any integration inputs (and push subscription inputs when newGlobals provided)
            ...(await this.loadIntegrationInputs(hogFunction, newGlobals, { throwIfMissingIn: ownInputs })),
        }

        // One budget for the whole invocation, so many small liquid leaves cannot add up to a stall.
        const liquidBudget = new LiquidRenderBudget()

        // Build a lookup of schema types for rendering and post-render coercion
        const schemaTypes: Record<string, string> = {}
        for (const schema of hogFunction.inputs_schema ?? []) {
            schemaTypes[schema.key] = schema.type
        }

        const _formatInput = async (input: CyclotronInputType, key: string): Promise<any> => {
            const templating = input.templating ?? 'hog'

            if (templating === 'liquid') {
                let value = input.value
                // `design` is the email editor's state and is never sent. Its JSON-encoded strings escape quotes
                // as \", which Liquid cannot parse. validation.py drops it the same way before it compiles hog.
                if (EMAIL_INPUT_TYPES.includes(schemaTypes[key]) && value?.design !== undefined) {
                    const { design: _design, ...rest } = value
                    value = rest
                }
                // A device shows push text literally, so an HTML entity would reach the person as written.
                const plainTextKeys = EMAIL_INPUT_TYPES.includes(schemaTypes[key])
                    ? new Set(EMAIL_PLAIN_TEXT_FIELDS.map((field) => `${key}.${field}`))
                    : hogFunction.template_id === NATIVE_PUSH_TEMPLATE_ID
                      ? new Set([key])
                      : undefined
                return formatLiquidInput(value, newGlobals, key, liquidBudget, plainTextKeys)
            }
            if (templating === 'hog' && input?.bytecode) {
                try {
                    return await formatHogInput(input.bytecode, newGlobals, key)
                } catch (error) {
                    throw withBytecodeContract(error, input.bytecode_contract)
                }
            }

            return input.value
        }

        // Add unsubscribe url if we have an email input here
        const emailInputSchema = hogFunction.inputs_schema?.find((input) => EMAIL_INPUT_TYPES.includes(input.type))
        const emailInput = hogFunction.inputs?.[emailInputSchema?.key ?? '']

        if (emailInputSchema && emailInput) {
            if (!this.recipientTokensService) {
                throw new Error('HogInputsService was constructed without messaging preference URL support')
            }
            // If we have an email value then we template it out to get the email address
            const emailValue = await _formatInput(emailInput, emailInputSchema.key)
            if (emailValue?.to?.email) {
                const recipient = { team_id: hogFunction.team_id, identifier: emailValue.to.email }
                newGlobals.unsubscribe_url = this.recipientTokensService.generatePreferencesUrl(
                    recipient,
                    preferencesTokenSource
                )
                newGlobals.unsubscribe_url_one_click = this.recipientTokensService.generateOneClickUnsubscribeUrl(
                    recipient,
                    preferencesTokenSource
                )
            }
        }

        const orderedInputs = Object.entries(inputs ?? {}).sort(([_k1, input1], [_k2, input2]) => {
            return (input1?.order ?? -1) - (input2?.order ?? -1)
        })

        for (const [key, input] of orderedInputs) {
            if (!input) {
                continue
            }

            let inputsResult = await _formatInput(input, key)

            // Safety net: coerce string results to booleans for boolean schema fields.
            // Handles edge cases where Liquid templating returns strings for boolean fields.
            if (schemaTypes[key] === 'boolean' && typeof inputsResult === 'string') {
                const lower = inputsResult.trim().toLowerCase()
                inputsResult = lower === 'true' || lower === '1'
            }

            newGlobals.inputs[key] = inputsResult
        }

        return newGlobals.inputs
    }

    public async buildInputsWithGlobals(
        hogFunction: HogFunctionType,
        globals: HogFunctionInvocationGlobals,
        additionalInputs?: Record<string, any>,
        preferencesTokenSource?: PreferencesTokenSource
    ): Promise<HogFunctionInvocationGlobalsWithInputs> {
        return {
            ...globals,
            inputs: await this.buildInputs(hogFunction, globals, additionalInputs, preferencesTokenSource),
        }
    }

    private resolvePushSubscriptionInputs(
        hogFunction: HogFunctionType,
        integrationInputs: Record<string, { value: Record<string, any> | Record<string, any>[] | null }>,
        newGlobals: HogFunctionInvocationGlobalsWithInputs
    ): Record<string, { value: string | null }> {
        const result: Record<string, { value: string | null }> = {}
        const personProperties = newGlobals.person?.properties

        const appIdentifier = getAppIdentifierForPush(integrationInputs)

        hogFunction.inputs_schema?.forEach((schema) => {
            if (schema.type !== 'push_subscription') {
                return
            }

            if (!appIdentifier) {
                logger.warn('🦔', '[HogInputsService] No push integration found for push subscription input', {
                    hogFunctionId: hogFunction.id,
                    hogFunctionName: hogFunction.name,
                    teamId: hogFunction.team_id,
                    inputKey: schema.key,
                })
                result[schema.key] = { value: null }
                return
            }

            const token = getDevicePushSubscriptionToken(personProperties, appIdentifier, this.encryptedFields)
            result[schema.key] = { value: token }
        })

        return result
    }

    // The fetch step swaps these values for the token wherever they appear in the request, so they must not be
    // something a request can contain unless the function read it from these inputs.
    private accessTokenPlaceholder(integrationId: number): string {
        const base = accessTokenPlaceholderBase(integrationId)
        return signedAccessTokenPlaceholder(base, this.encryptedFields.sign(ACCESS_TOKEN_PLACEHOLDER_PURPOSE, base))
    }

    public acceptedAccessTokenPlaceholders(integrationId: number): string[] {
        const base = accessTokenPlaceholderBase(integrationId)
        return this.encryptedFields
            .signatures(ACCESS_TOKEN_PLACEHOLDER_PURPOSE, base)
            .map((signature) => signedAccessTokenPlaceholder(base, signature))
    }

    public async loadIntegrationInputs(
        hogFunction: HogFunctionType,
        newGlobals?: HogFunctionInvocationGlobalsWithInputs,
        // Loading covers every mapping, so only the inputs this invocation uses may fail it.
        options?: { throwIfMissingIn?: HogFunctionType['inputs'] }
    ): Promise<Record<string, { value: Record<string, any> | Record<string, any>[] | null }>> {
        // Single integration inputs map a key to one id; integration_multi maps a key to a list of ids.
        const singleInputsToLoad: Record<string, number> = {}
        const multiInputsToLoad: Record<string, number[]> = {}

        const allSchemas = [
            ...(hogFunction.inputs_schema ?? []),
            ...(hogFunction.mappings?.flatMap((m) => m.inputs_schema ?? []) ?? []),
        ]
        const allInputs: Record<string, CyclotronInputType | null | undefined> = {
            ...hogFunction.inputs,
        }
        for (const mapping of hogFunction.mappings ?? []) {
            Object.assign(allInputs, mapping.inputs)
        }

        allSchemas.forEach((schema) => {
            const value = allInputs[schema.key]?.value
            if (schema.type === 'integration') {
                if (value && typeof value === 'number') {
                    singleInputsToLoad[schema.key] = value
                }
            } else if (schema.type === 'integration_multi') {
                if (Array.isArray(value)) {
                    const ids = value.filter((v): v is number => typeof v === 'number')
                    if (ids.length > 0) {
                        multiInputsToLoad[schema.key] = ids
                    }
                }
            }
        })

        const allIds = [...Object.values(singleInputsToLoad), ...Object.values(multiInputsToLoad).flat()]
        if (allIds.length === 0) {
            return {}
        }

        const integrations = await this.integrationManager.getMany(allIds)
        const returnInputs: Record<string, { value: Record<string, any> | Record<string, any>[] | null }> = {}

        // IMPORTANT: Check the team ID is correct — never resolve an integration from another team.
        const resolveIntegration = (id: number): Record<string, any> | null => {
            const integration = integrations[id]
            if (!integration || integration.team_id !== hogFunction.team_id) {
                return null
            }
            // A `posthog` connection holds its creator's grant into another project, and only the creator
            // may use it. A function runs for the whole team, so resolving one would hand that grant to
            // every member who can edit or test functions.
            if (integration.kind === 'posthog') {
                return null
            }
            if (isWarehouseSourceIntegration(integration)) {
                return null
            }
            return {
                $integration_id: integration.id,
                ...integration.config,
                ...integration.sensitive_config,
                ...(integration.sensitive_config.access_token || integration.config?.access_token
                    ? {
                          access_token: this.accessTokenPlaceholder(integration.id),
                          access_token_raw:
                              integration.sensitive_config.access_token ?? integration.config?.access_token,
                      }
                    : {}),
            }
        }

        Object.entries(singleInputsToLoad).forEach(([key, id]) => {
            // Without this the template runs with an empty connection, and a URL built from it fails far from the cause.
            const isUsed = options?.throwIfMissingIn?.[key]?.value === id
            if (isUsed && (!integrations[id] || integrations[id].team_id !== hogFunction.team_id)) {
                throw new HogConfigurationError(
                    `The integration for input '${key}' no longer exists. It may have been deleted. Select a connected integration for this input.`
                )
            }
            if (isUsed && isWarehouseSourceIntegration(integrations[id])) {
                throw new HogConfigurationError(warehouseIntegrationMessage(key))
            }
            returnInputs[key] = { value: resolveIntegration(id) }
        })

        Object.entries(multiInputsToLoad).forEach(([key, ids]) => {
            const usedValue = options?.throwIfMissingIn?.[key]?.value
            const usedIds = Array.isArray(usedValue) ? usedValue : []
            // Only classify this team's rows, so another team's IDs can't be probed through the error.
            if (
                ids.some(
                    (id) =>
                        usedIds.includes(id) &&
                        integrations[id]?.team_id === hogFunction.team_id &&
                        isWarehouseSourceIntegration(integrations[id])
                )
            ) {
                throw new HogConfigurationError(warehouseIntegrationMessage(key))
            }
            returnInputs[key] = { value: ids.map(resolveIntegration).filter((v) => v !== null) }
        })

        if (newGlobals) {
            const pushSubscriptionInputs = this.resolvePushSubscriptionInputs(hogFunction, returnInputs, newGlobals)
            Object.assign(returnInputs, pushSubscriptionInputs)
        }

        return returnInputs
    }
}

// A row with the `warehouse_source_type` key holds data warehouse source credentials. They are never Hog function
// inputs. This matches the Python check: the key blocks with any value, and a config that is not an object blocks too.
const isWarehouseSourceIntegration = (integration: { config?: unknown } | null | undefined): boolean => {
    const config = integration?.config
    if (config === null || config === undefined) {
        return false
    }
    if (typeof config !== 'object' || Array.isArray(config)) {
        return true
    }
    return 'warehouse_source_type' in config
}

const warehouseIntegrationMessage = (key: string): string =>
    `The integration for input '${key}' can't be used as a function input. Select a different integration for this input.`

export const formatHogInput = async (
    bytecode: any,
    globals: HogFunctionInvocationGlobalsWithInputs,
    key?: string
): Promise<any> => {
    // Similar to how we generate the bytecode by iterating over the values,
    // here we iterate over the object and replace the bytecode with the actual values
    // bytecode is indicated as an array beginning with ["_H"] (versions 1+) or ["_h"] (version 0)

    if (bytecode === null || bytecode === undefined) {
        return bytecode // Preserve null and undefined values
    }

    if (Array.isArray(bytecode) && (bytecode[0] === '_h' || bytecode[0] === '_H')) {
        const { execResult: result, error } = await execHog(bytecode, { globals })
        if (!result || error) {
            throw error ?? result?.error
        }
        if (!result?.finished) {
            // An uncaught hog exception comes back as an unfinished run with the error attached.
            // Other VM messages can echo an argument, and an argument can be a secret input.
            const message: string = result?.error?.message ?? ''
            const detail = message.startsWith('Global variable not found') ? `: ${message}` : ''
            // Only the kind travels, so the caller can classify the failure. The VM error itself stays
            // behind: a serialized cause chain would print its message, and that can hold a secret.
            const kind: unknown = result?.error?.kind
            const cause = isHogVMErrorKind(kind)
                ? new HogVMException(`Input field ${key} could not be evaluated`, kind)
                : undefined
            throw new Error(`Could not execute bytecode for input field: ${key}${detail}`, { cause })
        }
        return convertHogToJS(result.result)
    }

    if (Array.isArray(bytecode)) {
        return await Promise.all(bytecode.map((item) => formatHogInput(item, globals, key)))
    } else if (typeof bytecode === 'object' && bytecode !== null) {
        let ret: Record<string, any> = {}

        if (bytecode[EXTEND_OBJECT_KEY]) {
            const res = await formatHogInput(bytecode[EXTEND_OBJECT_KEY], globals, key)
            if (res && typeof res === 'object') {
                ret = {
                    ...res,
                }
            }
        }

        await Promise.all(
            Object.entries(bytecode).map(async ([subkey, value]) => {
                if (subkey === EXTEND_OBJECT_KEY) {
                    return
                }
                ret[subkey] = await formatHogInput(value, globals, key ? `${key}.${subkey}` : subkey)
            })
        )

        return ret
    }

    return bytecode
}

/**
 * A parse error or a refused filter fails the template on every event. Any other render error is
 * the template meeting this event's values, so another event may pass. A budget is a limit.
 */
const liquidErrorKind = (error: unknown): HogVMErrorKind => {
    const { name, message } = error instanceof Error ? error : { name: '', message: String(error) }
    if (message.includes('limit exceeded')) {
        return 'limit'
    }
    if (name === 'RenderError' && !message.includes('is not supported')) {
        return 'data'
    }
    return 'contract'
}

// A key is plain text when it or any object it sits in is listed, such as `data.url` under `data`.
const isPlainTextKey = (key: string | undefined, plainTextKeys: ReadonlySet<string> | undefined): boolean => {
    let current = key
    while (current && plainTextKeys) {
        if (plainTextKeys.has(current)) {
            return true
        }
        const dot = current.lastIndexOf('.')
        current = dot === -1 ? undefined : current.slice(0, dot)
    }
    return false
}

export const formatLiquidInput = (
    value: unknown,
    globals: HogFunctionInvocationGlobalsWithInputs,
    key?: string,
    budget: LiquidRenderBudget = new LiquidRenderBudget(),
    plainTextKeys?: ReadonlySet<string>
): any => {
    if (value === null || value === undefined) {
        return value
    }

    if (typeof value === 'string') {
        try {
            return LiquidRenderer.renderWithHogFunctionGlobals(value, globals, budget, {
                escapeHtml: !isPlainTextKey(key, plainTextKeys),
            })
        } catch (error) {
            // The renderer's message names the template line the owner has to fix, so it stays as is.
            const message = error instanceof Error ? error.message : String(error)
            throw new Error(message, { cause: new HogVMException(message, liquidErrorKind(error)) })
        }
    }

    if (Array.isArray(value)) {
        return value.map((item) => formatLiquidInput(item, globals, key, budget, plainTextKeys))
    }

    if (typeof value === 'object' && value !== null) {
        return Object.fromEntries(
            Object.entries(value).map(([key2, value]) => [
                key2,
                formatLiquidInput(value, globals, key ? `${key}.${key2}` : key2, budget, plainTextKeys),
            ])
        )
    }

    return value
}

/**
 * Finds the app identifier (project_id for Firebase, bundle_id for APNS) from resolved
 * integration inputs. Used to key the $device_push_subscription_<appIdentifier> person property.
 */
export function getAppIdentifierForPush(
    integrationInputs: Record<string, { value: Record<string, any> | Record<string, any>[] | null }>
): string | null {
    for (const input of Object.values(integrationInputs)) {
        const value = input?.value
        if (!value || Array.isArray(value)) {
            continue
        }
        const appId = value.project_id ?? value.bundle_id
        if (typeof appId === 'string' && appId) {
            return appId
        }
    }
    return null
}