:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { DateTime } from 'luxon'
import { Summary } from 'prom-client'
import { gunzip, gzip } from 'zlib'
import { parseJSON } from '~/common/utils/json-parse'
import { sanitizeForUTF8 } from '~/common/utils/strings'
import { UUIDT, castTimestampOrNow, clickHouseTimestampToDateTime } from '~/common/utils/utils'
import { ClickHouseTimestamp, ISOTimestamp, RawClickHouseEvent, Team, TimestampFormat } from '../types'
import { CdpInternalEvent } from './schema'
import { HogFunctionInvocationGlobals, HogFunctionType, LogEntry, LogEntrySerialized, MinimalLogEntry } from './types'
// ID of functions that are hidden from normal users and used by us for special testing
// For example, transformations use this to only run if in comparison mode
export const CDP_TEST_ID = '[CDP-TEST-HIDDEN]'
export const MAX_LOG_LENGTH = 10000
const TRUNCATION_SUFFIX = '... (truncated)'
const REDACTED = '***REDACTED***'
// Sync with person.py and constants.tsx
export const PERSON_DEFAULT_DISPLAY_NAME_PROPERTIES = ['email', 'name', 'username']
export const getPersonDisplayName = (team: Team, distinctId: string, properties: Record<string, any>): string => {
const personDisplayNameProperties = team.person_display_name_properties ?? PERSON_DEFAULT_DISPLAY_NAME_PROPERTIES
const customPropertyKey = personDisplayNameProperties.find((x) => properties?.[x])
const propertyIdentifier = customPropertyKey ? properties[customPropertyKey] : undefined
const customIdentifier: string =
typeof propertyIdentifier !== 'string' ? JSON.stringify(propertyIdentifier) : propertyIdentifier
return (customIdentifier || String(distinctId))?.trim()
}
// TRICKY: the timestamp can be an ISO one, for example when it comes from the test api, so both
// formats are accepted. Returns null rather than throwing, because a test-invocation body can carry
// a `clickhouse_event` that is not an event at all, such as the `{}` a caller sends for an optional
// field it has no value for.
export function parseClickHouseEventTimestamp(timestamp: unknown): ISOTimestamp | null {
if (typeof timestamp !== 'string') {
return null
}
if (DateTime.fromISO(timestamp).isValid) {
return timestamp as ISOTimestamp
}
const parsed = clickHouseTimestampToDateTime(timestamp as ClickHouseTimestamp)
return parsed.isValid ? (parsed.toISO() as ISOTimestamp) : null
}
export function isConvertibleClickHouseEvent(event: unknown): event is RawClickHouseEvent {
return (
!!event &&
typeof event === 'object' &&
parseClickHouseEventTimestamp((event as RawClickHouseEvent).timestamp) !== null
)
}
// that we can keep to as a contract
export function convertToHogFunctionInvocationGlobals(
event: RawClickHouseEvent,
team: Team,
siteUrl: string
): HogFunctionInvocationGlobals {
const properties = event.properties ? parseJSON(event.properties) : {}
const projectUrl = `${siteUrl}/project/${team.id}`
let person: HogFunctionInvocationGlobals['person']
if (event.person_id) {
const personProperties = event.person_properties ? parseJSON(event.person_properties) : {}
const personDisplayName = getPersonDisplayName(team, event.distinct_id, personProperties)
person = {
id: event.person_id,
properties: personProperties,
name: personDisplayName,
url: `${projectUrl}/person/${encodeURIComponent(event.distinct_id)}`,
}
}
const eventTimestamp = parseClickHouseEventTimestamp(event.timestamp)!
const eventCapturedAt = parseClickHouseEventTimestamp(event.captured_at)
const eventUrl = `${projectUrl}/events/${encodeURIComponent(event.uuid)}/${encodeURIComponent(eventTimestamp)}`
const context: HogFunctionInvocationGlobals = {
project: {
id: team.id,
name: team.name,
url: projectUrl,
},
event: {
uuid: event.uuid,
event: event.event!,
elements_chain: event.elements_chain,
distinct_id: event.distinct_id,
properties,
timestamp: eventTimestamp,
captured_at: eventCapturedAt,
// The events page reads flag calls from flag_evaluations only when the link names the event.
// On flag_evaluations_mode 2 a flag call has no events row, so a link without the name finds nothing.
url:
event.event === '$feature_flag_called'
? `${eventUrl}?event=${encodeURIComponent(event.event)}`
: eventUrl,
},
person,
}
return context
}
export function convertBatchHogFlowRequestToHogFunctionInvocationGlobals({
team,
personId,
siteUrl,
}: {
team: Team
personId: string
siteUrl: string
}): HogFunctionInvocationGlobals {
const projectUrl = `${siteUrl}/project/${team.id}`
const person: HogFunctionInvocationGlobals['person'] = {
id: personId,
properties: {},
name: '',
url: `${projectUrl}/person/${encodeURIComponent(personId)}`,
}
const context: HogFunctionInvocationGlobals = {
project: {
id: team.id,
name: team.name,
url: projectUrl,
},
event: {
event: '$batch_hog_flow_invocation',
properties: {},
uuid: new UUIDT().toString(),
distinct_id: '', // Not applicable for batch processing but left here for compatibility
elements_chain: '',
timestamp: DateTime.now().toISO(),
url: '',
},
person,
}
return context
}
export function convertAccountBatchHogFlowRequestToHogFunctionInvocationGlobals({
team,
externalId,
groupType,
siteUrl,
}: {
team: Team
externalId: string
groupType: string
siteUrl: string
}): HogFunctionInvocationGlobals {
const projectUrl = `${siteUrl}/project/${team.id}`
const context: HogFunctionInvocationGlobals = {
project: {
id: team.id,
name: team.name,
url: projectUrl,
},
event: {
event: '$batch_hog_flow_invocation',
// $groups drives the worker's group hydration, so account actions defaulting to
// {groups.<type>.id} resolve without any account-specific plumbing.
properties: { $groups: { [groupType]: externalId } },
uuid: new UUIDT().toString(),
// The account's group key doubles as the invocation's distinct_id so
// invocation_results are filterable per account. Account runs carry no person;
// the hogflow worker skips person resolution for account audiences.
distinct_id: externalId,
elements_chain: '',
timestamp: DateTime.now().toISO(),
url: '',
},
}
return context
}
export function convertInternalEventToHogFunctionInvocationGlobals(
data: CdpInternalEvent,
team: Team,
siteUrl: string
): HogFunctionInvocationGlobals {
const projectUrl = `${siteUrl}/project/${team.id}`
let person: HogFunctionInvocationGlobals['person']
if (data.person) {
const personDisplayName = getPersonDisplayName(team, data.event.distinct_id, data.person.properties)
person = {
id: data.person.id,
properties: data.person.properties,
name: personDisplayName,
url: data.person.url ?? '',
}
}
let properties = data.event.properties
// KLUDGE: spread the properties of the exception event that caused the internal issue event
// so those properties can be used to filter CDP destinations for error tracking alerts
if (
isInternalErrorTrackingEvent(data.event) &&
'exception_props' in properties &&
typeof properties.exception_props === 'object'
) {
properties = { ...properties.exception_props, ...properties }
delete properties.exception_props
}
const context: HogFunctionInvocationGlobals = {
project: {
id: team.id,
name: team.name,
url: projectUrl,
},
event: {
uuid: data.event.uuid,
event: data.event.event,
elements_chain: '', // Not applicable but left here for compatibility
distinct_id: data.event.distinct_id,
properties: properties,
timestamp: data.event.timestamp,
captured_at: null, // Not applicable for internal events
url: data.event.url ?? '',
},
person,
}
return context
}
export const gzipObject = async <T extends object>(object: T): Promise<string> => {
const payload = JSON.stringify(object)
const buffer = await new Promise<Buffer>((res, rej) =>
gzip(payload, (err, result) => (err ? rej(err) : res(result)))
)
const res = buffer.toString('base64')
// NOTE: Base64 encoding isn't as efficient but we would need to change the kafka producer/consumers to use ucs2 or something
// as well in order to support binary data better
return res
}
export const unGzipObject = async <T extends object>(data: string): Promise<T> => {
const res = await new Promise<Buffer>((res, rej) =>
gunzip(Buffer.from(data, 'base64'), (err, result) => (err ? rej(err) : res(result)))
)
return parseJSON(res.toString())
}
export const fixLogDeduplication = (logs: LogEntry[]): LogEntrySerialized[] => {
const preparedLogs: LogEntrySerialized[] = []
const sortedLogs = logs.sort((a, b) => a.timestamp.toMillis() - b.timestamp.toMillis())
if (sortedLogs.length === 0) {
return []
}
// Start with a timestamp that is guaranteed to be before the first log entry
let previousTimestamp = sortedLogs[0].timestamp.minus(1)
sortedLogs.forEach((logEntry) => {
// TRICKY: The clickhouse table dedupes logs with the same timestamp - we need to ensure they are unique by simply plus-ing 1ms
// if the timestamp is the same as the previous one
// Changes the caller's entry in place, not only the copy below. HogFunctionMonitoringService relies on this to
// send the same timestamps to the logs product, and its test fails by 1 ms if this stops.
if (logEntry.timestamp <= previousTimestamp) {
logEntry.timestamp = previousTimestamp.plus(1)
}
previousTimestamp = logEntry.timestamp
const sanitized: LogEntrySerialized = {
...logEntry,
timestamp: castTimestampOrNow(logEntry.timestamp, TimestampFormat.ClickHouse),
}
preparedLogs.push(sanitized)
})
return preparedLogs
}
export function isLegacyPluginHogFunction(hogFunction: Pick<HogFunctionType, 'template_id'>): boolean {
return hogFunction.template_id?.startsWith('plugin-') ?? false
}
export function isSegmentPluginHogFunction(hogFunction: Pick<HogFunctionType, 'template_id'>): boolean {
return hogFunction.template_id?.startsWith('segment-') ?? false
}
export function isNativeHogFunction(hogFunction: Pick<HogFunctionType, 'template_id'>): boolean {
return hogFunction.template_id?.startsWith('native-') ?? false
}
export function isInternalErrorTrackingEvent(event: CdpInternalEvent['event']): boolean {
return [
'$error_tracking_issue_created',
'$error_tracking_issue_reopened',
'$error_tracking_issue_spiking',
].includes(event.event)
}
export function filterExists<T>(value: T): value is NonNullable<T> {
return Boolean(value)
}
// Keys whose value is a credential however the request built it, for example a Basic `Authorization`
// header that a destination derives from an API key. No collected input value matches a derived value.
const LOGGED_CREDENTIAL_KEY =
/authorization|cookie|token|secret|password|passcode|credential|api[-_]?key|private[-_]?key|signing[-_]?key/i
// Header names that carry a credential. Matched against the keys of a dictionary input, so a
// free-form headers map still gets its credential masked without the whole map being treated as
// secret (which would hide ordinary headers like Content-Type from the person configuring it).
const CREDENTIAL_HEADER_NAMES =
/^(authorization|proxy-authorization|cookie|x-api-key|api-?key|x-auth-token|auth-?token|access-?token|x-auth|token|secret|x-secret)$/i
/**
* Every configured secret for a hog function, so callers can mask them out of anything they surface.
*
* A destination's error text is not safe just because we wrote the format string: it routinely
* embeds the third party's response body, and an API that rejects a credential tends to quote the
* credential back. Anything built from a response body has to go through `redactSensitiveValues`
* with this list before it reaches a log, an error, or ClickHouse.
*/
export const getSensitiveValues = (hogFunction: HogFunctionType, inputs: Record<string, any>): string[] => {
const values: string[] = []
const collectStringValues = (obj: any, depth = 0): void => {
// The depth limit stops a self-referencing object from recursing forever.
if (!obj || typeof obj !== 'object' || depth > 10) {
return
}
// Assume the values are the sensitive parts. Some integrations nest them, for example a
// Google Cloud key file under `key_info`, so nested objects and arrays are collected too.
Object.values(obj).forEach((val: any) => {
if (typeof val === 'string') {
values.push(val)
} else {
collectStringValues(val, depth + 1)
}
})
}
// A webhook's `headers` is free-form, so it is not marked secret, but it is where a credential
// most often sits — and a webhook whose credential is rejected is exactly the case that gets it
// quoted back. Mask values under header names that carry one, whatever the secret flag says.
// Every entry of a secret dictionary is a credential, whatever its header name.
const collectCredentialHeaders = (obj: any, anyHeaderName: boolean): void => {
if (!obj || typeof obj !== 'object') {
return
}
Object.entries(obj).forEach(([key, val]) => {
if (typeof val !== 'string' || (!anyHeaderName && !CREDENTIAL_HEADER_NAMES.test(key.trim()))) {
return
}
values.push(val)
// Sent as "Bearer abc" but usually quoted back as bare "abc", so mask both forms.
const withoutScheme = val.replace(/^(bearer|basic|token)\s+/i, '')
if (withoutScheme !== val) {
values.push(withoutScheme)
}
})
}
hogFunction.inputs_schema?.forEach((schema) => {
if (schema.type === 'dictionary') {
collectCredentialHeaders(inputs[schema.key], !!schema.secret)
}
// A function keeps the schema it was saved with, so a credential input that a later template
// version marks secret is still `secret: false` on older functions.
if (
schema.secret ||
LOGGED_CREDENTIAL_KEY.test(schema.key) ||
schema.type === 'integration' ||
schema.type === 'integration_multi' ||
schema.type === 'push_subscription'
) {
const value = inputs[schema.key]
if (typeof value === 'string') {
values.push(value)
} else if (schema.type === 'integration_multi' && Array.isArray(value)) {
// integration_multi resolves to an array of integration objects, each carrying its own
// sensitive_config (e.g. APNs signing_key, FCM access_token_raw) — mask every one.
value.forEach((item) => collectStringValues(item))
} else if (
(schema.type === 'dictionary' ||
schema.type === 'integration' ||
schema.type === 'push_subscription') &&
typeof value === 'object'
) {
collectStringValues(value)
}
}
})
// We don't want to add "REDACTED" for empty strings
return values.filter((v) => v.trim())
}
/**
* Sensitive values from the function config alone, before any template resolves. Covers an error
* raised while the inputs are still being built, when the resolved values do not exist yet.
*/
export const getConfiguredSensitiveValues = (hogFunction: HogFunctionType): string[] => {
const configured = Object.fromEntries(
Object.entries({ ...hogFunction.inputs, ...hogFunction.encrypted_inputs }).map(([key, input]) => [
key,
input?.value,
])
)
return getSensitiveValues(hogFunction, configured)
}
export const redactSensitiveValues = (message: string, sensitiveValues?: string[]): string => {
// Callers pass `err.message` straight from a catch, where `err` is `any` and need not be an
// Error at all, so a non-string reaches this despite the signature. Hand it back untouched
// rather than throwing inside the code path that is reporting someone else's failure.
if (!message || typeof message !== 'string' || !sensitiveValues?.length) {
return message
}
const variants = new Set<string>()
sensitiveValues.forEach((sensitiveValue) => {
if (typeof sensitiveValue !== 'string' || !sensitiveValue) {
return
}
variants.add(sensitiveValue)
// Logged objects are JSON-encoded, which escapes quotes, backslashes and newlines, so a secret
// such as a PEM private key no longer matches its raw form inside the message.
variants.add(JSON.stringify(sensitiveValue).slice(1, -1))
})
if (!variants.size) {
return message
}
// Replace every value in one pass over the original text, longest first. Replacing the values one
// after another rescans the markers that earlier passes inserted, so a value that occurs inside
// the marker, such as `*`, multiplies the message length on every pass.
const pattern = new RegExp(
[...variants]
.sort((a, b) => b.length - a.length)
.map((variant) => variant.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'))
.join('|'),
'g'
)
return message.replace(pattern, REDACTED)
}
// Redacts each string before the outer JSON encoding. A value that is already a JSON string, such as a
// request body, is otherwise escaped a second time and no longer matches the escaped secret.
const serializeRedactedLogArg = (arg: unknown, sensitiveValues: string[]): unknown => {
if (typeof arg === 'string') {
return arg
}
return JSON.stringify(arg, (key, value) => {
if (typeof value !== 'string') {
return value
}
return key && LOGGED_CREDENTIAL_KEY.test(key) ? REDACTED : redactSensitiveValues(value, sensitiveValues)
})
}
// Masks secrets in a caught error but keeps the error itself, so `instanceof` retry checks and
// `String(error)` behave as before.
export const redactError = (error: any, sensitiveValues: string[]): any => {
if (typeof error === 'string') {
return redactSensitiveValues(error, sensitiveValues)
}
if (error instanceof Error) {
const message = redactSensitiveValues(error.message, sensitiveValues)
try {
error.message = message
} catch {
return new Error(message)
}
}
return error
}
export const sanitizeLogMessage = (args: any[], sensitiveValues?: string[], maxLength = MAX_LOG_LENGTH): string => {
let message = redactSensitiveValues(
args.map((arg) => (typeof arg !== 'string' ? JSON.stringify(arg) : arg)).join(', '),
sensitiveValues
)
let truncateAt = maxLength
// Check if we're in the middle of a surrogate pair
if (truncateAt > 0 && truncateAt < message.length + TRUNCATION_SUFFIX.length) {
const charAtTruncate = message.charCodeAt(truncateAt)
const charBeforeTruncate = message.charCodeAt(truncateAt - 1)
// If we're about to cut after a high surrogate or before a low surrogate
if ((charBeforeTruncate & 0xfc00) === 0xd800 || (charAtTruncate & 0xfc00) === 0xdc00) {
// Move back to avoid cutting through the surrogate pair
truncateAt--
// If we moved back and are still at a high surrogate, move back one more
if (truncateAt > 0 && (message.charCodeAt(truncateAt - 1) & 0xfc00) === 0xd800) {
truncateAt--
}
}
message = sanitizeForUTF8(message.slice(0, truncateAt) + TRUNCATION_SUFFIX)
}
return message
}
export const logEntry = (level: 'debug' | 'warn' | 'error' | 'info', ...args: any[]) => {
return {
level,
timestamp: DateTime.now(),
message: sanitizeLogMessage(args),
}
}
export const createAddLogFunction = (logs: MinimalLogEntry[], sensitiveValues?: string[]) => {
return (level: 'debug' | 'warn' | 'error' | 'info', ...args: any[]) => {
const logArgs = sensitiveValues ? args.map((arg) => serializeRedactedLogArg(arg, sensitiveValues)) : args
logs.push({
level,
timestamp: DateTime.now(),
message: sanitizeLogMessage(logArgs, sensitiveValues),
})
}
}
export const destinationE2eLagMsSummary = new Summary({
name: 'destination_e2e_lag_ms',
help: 'Time difference in ms between event capture time and destination finishing time',
percentiles: [0.5, 0.9, 0.95, 0.99],
})